Configurable reduce limit threshold and ratio

rel/overlay/etc/default.ini

@@ -563,6 +563,11 @@ authentication_db = _users
 ; think you're hitting reduce_limit with a "good" reduce function, please let
 ; us know on the mailing list so we can fine tune the heuristic.
 ;reduce_limit = true
+; Don't log/crash a reduce if the result is less than this number of bytes;
+;reduce_limit_threshold = 5000
+; Don't log/crash a reduce if the result is less than the ratio multiplied
+; by input size
+;reduce_limit_ratio = 2.0
 ;os_process_limit = 100
 ;os_process_idle_limit = 300
 ;os_process_soft_limit = 100

share/server/views.js

@@ -36,9 +36,9 @@ var Views = (function() {
     var reduce_line = JSON.stringify(reductions);
     var reduce_length = reduce_line.length;
     var input_length =  State.line_length - code_size
-    // TODO make reduce_limit config into a number
     if (State.query_config && State.query_config.reduce_limit &&
-          reduce_length > 4096 && ((reduce_length * 2) > input_length)) {
+        reduce_length > State.query_config.reduce_limit_threshold &&
+        ((reduce_length * State.query_config.reduce_limit_ratio) > input_length)) {
       var log_message = [
           "Reduce output must shrink more rapidly:",
           "input size:", input_length,

src/couch/src/couch_proc_manager.erl

@@ -732,6 +732,8 @@ remove_waiting_client(#client{wait_key = Key}) ->
 get_proc_config() ->
     {[
         {<<"reduce_limit">>, get_reduce_limit()},
+        {<<"reduce_limit_threshold">>, couch_query_servers:reduce_limit_threshold()},
+        {<<"reduce_limit_ratio">>, couch_query_servers:reduce_limit_ratio()},
         {<<"timeout">>, get_os_process_timeout()}
     ]}.
 

src/couch/src/couch_query_servers.erl

@@ -19,6 +19,7 @@
 -export([filter_view/4]).
 -export([finalize/2]).
 -export([rewrite/3]).
+-export([reduce_limit_threshold/0, reduce_limit_ratio/0]).
 
 -export([with_ddoc_proc/3, proc_prompt/2, ddoc_prompt/4, ddoc_proc_prompt/3, json_doc/1]).
 
@@ -278,7 +279,7 @@ sum_arrays(Else, _) ->
     throw_sum_error(Else).
 
 check_sum_overflow(InSize, OutSize, Sum) ->
-    Overflowed = OutSize > 4906 andalso OutSize * 2 > InSize,
+    Overflowed = OutSize > reduce_limit_threshold() andalso OutSize * reduce_limit_ratio() > InSize,
     case config:get("query_server_config", "reduce_limit", "true") of
         "true" when Overflowed ->
             Msg = log_sum_overflow(InSize, OutSize),
@@ -302,6 +303,12 @@ log_sum_overflow(InSize, OutSize) ->
     couch_log:error(Msg, []),
     Msg.
 
+reduce_limit_threshold() ->
+    config:get_integer("query_server_config", "reduce_limit_threshold", 5000).
+
+reduce_limit_ratio() ->
+    config:get_float("query_server_config", "reduce_limit_ratio", 2.0).
+
 builtin_stats(_, []) ->
     {0, 0, 0, 0, 0};
 builtin_stats(_, [[_, First] | Rest]) ->

src/couch/test/eunit/couch_query_servers_tests.erl

@@ -23,7 +23,9 @@ setup() ->
     Ctx.
 
 teardown(Ctx) ->
-    config:delete("query_server_config", "reduce_limit", "true", false),
+    config:delete("query_server_config", "reduce_limit", true),
+    config:delete("query_server_config", "reduce_limit_threshold", true),
+    config:delete("query_server_config", "reduce_limit_ratio", true),
     config:delete("log", "level", false),
     test_util:stop_couch(Ctx),
     meck:unload().
@@ -37,6 +39,8 @@ query_server_limits_test_() ->
             fun teardown/1,
             [
                 ?TDEF_FE(builtin_should_return_error_on_overflow),
+                ?TDEF_FE(builtin_should_not_return_error_with_generous_overflow_threshold),
+                ?TDEF_FE(builtin_should_not_return_error_with_generous_overflow_ratio),
                 ?TDEF_FE(builtin_should_return_object_on_log),
                 ?TDEF_FE(builtin_should_return_object_on_false),
                 ?TDEF_FE(js_reduce_should_return_error_on_overflow),
@@ -55,6 +59,22 @@ builtin_should_return_error_on_overflow(_) ->
     ?assertMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result),
     ?assert(meck:called(couch_log, error, '_')).
 
+builtin_should_not_return_error_with_generous_overflow_threshold(_) ->
+    config:set("query_server_config", "reduce_limit", "true", false),
+    config:set_integer("query_server_config", "reduce_limit_threshold", 1000000, false),
+    meck:reset(couch_log),
+    KVs = gen_sum_kvs(),
+    {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs),
+    ?assertNotMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result).
+
+builtin_should_not_return_error_with_generous_overflow_ratio(_) ->
+    config:set("query_server_config", "reduce_limit", "true", false),
+    config:set_float("query_server_config", "reduce_limit_ratio", 0.1, false),
+    meck:reset(couch_log),
+    KVs = gen_sum_kvs(),
+    {ok, [Result]} = couch_query_servers:reduce(<<"foo">>, [<<"_sum">>], KVs),
+    ?assertNotMatch({[{<<"error">>, <<"builtin_reduce_error">>} | _]}, Result).
+
 builtin_should_return_object_on_log(_) ->
     config:set("query_server_config", "reduce_limit", "log", false),
     meck:reset(couch_log),

src/docs/src/config/query-servers.rst

@@ -150,6 +150,16 @@ Query Servers Configuration
         option since main propose of `reduce` functions is to *reduce* the
         input.
 
+    .. config:option:: reduce_limit_threshold :: Reduce limit threshold
+
+        The number of bytes a reduce result must exceed to trigger the ``reduce_limit``
+        control. Defaults to 5000.
+
+    .. config:option:: reduce_limit_ratio :: Reduce limit ratio
+
+        The ratio of input/output that must be exceeded to trigger the ``reduce_limit``
+        control. Defaults to 2.0.
+
 .. _config/native_query_servers:
 
 Native Erlang Query Server