Handle more non-determinism in update, filter and vdus

Expand non-determinism handling to update handlers, filters, and vdus.

Author: nickva

src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl

@@ -210,13 +210,16 @@ process_ddoc(#st{} = St, DbName, #doc{} = DDoc0) ->
                 Views = maps:get(?VIEWS, DDoc, undefined),
                 Clouseau = maps:get(?CLOUSEAU, DDoc, undefined),
                 Nouveau = maps:get(?NOUVEAU, DDoc, undefined),
+                Filters = maps:get(?FILTERS, DDoc, undefined),
+                Updates = maps:get(?UPDATES, DDoc, undefined),
+                Vdu = maps:get(?VDU, DDoc, undefined),
                 lib_load(St1, Views),
-                views_load(St1, valid_views(Views)),
+                views_load(St1, views(Views)),
                 clouseau_load(St1, indexes(Clouseau)),
                 nouveau_load(St1, indexes(Nouveau)),
-                filters_load(St1, maps:get(?FILTERS, DDoc, undefined)),
-                updates_load(St1, maps:get(?UPDATES, DDoc, undefined)),
-                vdu_load(St1, maps:get(?VDU, DDoc, undefined)),
+                filters_load(St1, filters(Filters)),
+                updates_load(St1, updates(Updates)),
+                vdu_load(St1, vdu(Vdu)),
                 St2 = start_or_reset_procs(St1),
                 teach_ddoc_validate(St2, DDocId, DDoc),
                 St2#st{ddocs = DDocs#{DDocId => DDoc}}
@@ -239,11 +242,11 @@ process_ddoc_functions(#st{} = St, Db, DocId, JsonDoc) ->
     DDocFun = fun(DDocId, #{} = DDoc) ->
         try
             Filters = maps:get(?FILTERS, DDoc, undefined),
-            filter_doc_validate(St, DDocId, Filters, JsonDoc),
             VDU = maps:get(?VDU, DDoc, undefined),
-            vdu_doc_validate(St, DDocId, VDU, JsonDoc),
             Updates = maps:get(?UPDATES, DDoc, undefined),
-            update_doc_validate(St, DDocId, Updates, JsonDoc)
+            filter_doc_validate(St, DDocId, filters(Filters), JsonDoc),
+            vdu_doc_validate(St, DDocId, vdu(VDU), JsonDoc),
+            update_doc_validate(St, DDocId, updates(Updates), JsonDoc)
         catch
             throw:{validate, Error} ->
                 Meta = #{sid => SId, db => Db, ddoc => DDocId, doc => DocId},
@@ -286,7 +289,7 @@ views_validate(DDocId, #{?VIEWS := Views}, {Db, #st{} = St0}) when
     #st{sid = SId, docs = Docs} = St,
     try
         lib_load(St, Views),
-        ViewList = lists:sort(maps:to_list(valid_views(Views))),
+        ViewList = lists:sort(maps:to_list(views(Views))),
         case ViewList of
             [_ | _] ->
                 Fun = fun({Name, #{?MAP := Src}}) -> add_fun_load(St, Name, Src) end,
@@ -475,7 +478,7 @@ start_or_reset_sm_proc(#st{sm_proc = #proc{} = Proc} = St) ->
             start_or_reset_sm_proc(St#st{sm_proc = undefined})
     end.
 
-valid_views(#{} = Views) ->
+views(#{} = Views) ->
     Fun = fun
         (?LIB, _) ->
             false;
@@ -487,7 +490,7 @@ valid_views(#{} = Views) ->
             false
     end,
     maps:filter(Fun, Views);
-valid_views(_) ->
+views(_) ->
     #{}.
 
 indexes(#{} = Indexes) ->
@@ -504,6 +507,32 @@ indexes(#{} = Indexes) ->
 indexes(_) ->
     #{}.
 
+updates(#{} = Updates) ->
+    Fun = fun
+        (<<_/binary>>, <<FunSrc/binary>>) -> no_indeterminism(FunSrc);
+        (_, _) -> false
+    end,
+    maps:filter(Fun, Updates);
+updates(_) ->
+    #{}.
+
+filters(#{} = Filters) ->
+    Fun = fun
+        (<<_/binary>>, <<FunSrc/binary>>) -> no_indeterminism(FunSrc);
+        (_, _) -> false
+    end,
+    maps:filter(Fun, Filters);
+filters(_) ->
+    #{}.
+
+vdu(<<FunSrc/binary>>) ->
+    case no_indeterminism(FunSrc) of
+        true -> FunSrc;
+        false -> undefined
+    end;
+vdu(_) ->
+    undefined.
+
 % Math.random(), Date.now() or new Date() will always show as false postives
 %
 no_indeterminism(<<FunSrc/binary>>) ->

src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl

@@ -25,6 +25,7 @@ couch_quickjs_scanner_plugin_test_() ->
             ?TDEF_FE(t_filter_map, 10),
             ?TDEF_FE(t_map_only, 10),
             ?TDEF_FE(t_vdu_only, 10),
+            ?TDEF_FE(t_non_deterministic_vdu, 10),
             ?TDEF_FE(t_filter_only, 10),
             ?TDEF_FE(t_filter_with_expected_error, 10),
             ?TDEF_FE(t_empty_ddoc, 10),
@@ -205,6 +206,39 @@ t_vdu_only({_, DbName}) ->
             ok
     end.
 
+t_non_deterministic_vdu({_, DbName}) ->
+    ok = add_doc(DbName, ?DDOC1, ddoc_non_deterministic_vdu(#{})),
+    meck:reset(couch_scanner_server),
+    meck:reset(?PLUGIN),
+    config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+    wait_exit(10000),
+    ?assertEqual(1, num_calls(start, 2)),
+    case couch_server:with_spidermonkey() of
+        true ->
+            ?assertEqual(1, num_calls(complete, 1)),
+            ?assertEqual(2, num_calls(checkpoint, 1)),
+            ?assertEqual(1, num_calls(db, ['_', DbName])),
+            ?assertEqual(1, num_calls(ddoc, ['_', DbName, '_'])),
+            ?assert(num_calls(shards, 2) >= 1),
+            DbOpenedCount = num_calls(db_opened, 2),
+            ?assert(DbOpenedCount >= 2),
+            ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])),
+            ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])),
+            ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])),
+            ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])),
+            ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])),
+            ?assert(num_calls(doc, 3) >= 5),
+            DbClosingCount = num_calls(db_closing, 2),
+            ?assertEqual(DbOpenedCount, DbClosingCount),
+            ?assertEqual(0, couch_stats:sample([couchdb, query_server, process_error_exits])),
+            ?assertEqual(0, couch_stats:sample([couchdb, query_server, process_errors])),
+            ?assertEqual(0, couch_stats:sample([couchdb, query_server, process_exits])),
+            % start, complete and no vdu warning because it has a Math.random(), so 2 total
+            ?assertEqual(2, log_calls(warning));
+        false ->
+            ok
+    end.
+
 t_filter_only({_, DbName}) ->
     ok = add_doc(DbName, ?DDOC1, ddoc_filter(#{})),
     meck:reset(couch_scanner_server),
@@ -463,6 +497,19 @@ ddoc_vdu(Doc) ->
         >>
     }.
 
+ddoc_non_deterministic_vdu(Doc) ->
+    Doc#{
+        validate_doc_update => <<
+            "function(newdoc, olddoc, userctx, sec){\n"
+            " if (Math.random() > 0.5) {\n"
+            "    throw('forbidden');\n"
+            " } else {\n"
+            "    return true\n"
+            " }\n"
+            "}"
+        >>
+    }.
+
 ddoc_filter(Doc) ->
     Doc#{
         filters => #{
@@ -481,7 +528,8 @@ ddoc_filter(Doc) ->
                 "    if (doc.a == req.query.foo) {return true;} \n"
                 "    return false; \n"
                 "}\n"
-            >>
+            >>,
+            f_non_deterministic => <<"function(doc, req) {return new Date();}}">>
         }
     }.
 
@@ -661,7 +709,8 @@ ddoc_update(Doc) ->
                 "  }\n"
                 "}"
             >>,
-            u3 => <<"function(doc, req) {throw('Potato')}">>
+            u3 => <<"function(doc, req) {throw('Potato')}">>,
+            u4_non_deterministic => <<"function(doc, req) {throw(Date.now())}">>
         }
     }.