Add new chaos test suite.

js/client/modules/@arangodb/testsuites/chaos.js

@@ -0,0 +1,103 @@
+/* jshint strict: false, sub: true */
+/* global */
+'use strict';
+
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License")
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Manuel Pöter
+////////////////////////////////////////////////////////////////////////////////
+
+const functionsDocumentation = {
+  'chaos': 'chaos tests'
+};
+const optionsDocumentation = [];
+
+const _ = require('lodash');
+const tu = require('@arangodb/testutils/test-utils');
+
+const testPaths = {
+  'chaos': [ tu.pathForTesting('client/chaos') ],
+};
+
+function chaos (options) {
+  let testCasesWithConfigs = {};
+  let testCases = tu.scanTestPaths(testPaths.chaos, options);
+
+  // The chaos test suite is parameterized and each configuration runs 5min.
+  // For the nightly tests we want to run a large number of possible parameter
+  // combinations, but each file has a runtime limit of 15min and ATM the test
+  // system is not designed to allow a test file to be run multiple times with
+  // different configurations.
+  // The hacky solution here is to build up a map of test cases with their respective
+  // configurations and have n copies of the test file in the test case list,
+  // where n is the number of configurations for this test. The new `preRun`
+  // callback function is called before each testCase execution. At that point we
+  // pop the next config from this test case's config list and set the global
+  // variable `currentTestConfig` which is then used in the tests.
+  // To allow the test cases to define the possible configs I introduced the
+  // possibility to write test cases as modules. A jsunity test file that contains
+  // "test-module-" in its filename is not executed directly, but instead must
+  // export a "run" function that is called. Such a module can optionally define
+  // a function "getConfigs" which must return an array of configurations.
+
+  testCases = _.flatMap(testCases, testCase => {
+    if (testCase.includes("test-module-")) {
+      const configProvider = require(testCase).getConfigs;
+      if (configProvider) {
+        const configs = configProvider();
+        if (!Array.isArray(configs)) {
+          throw "test case module " + testCase + " does not provide config list";
+        }
+        testCasesWithConfigs[testCase] = configs;
+        return Array(configs.length).fill(testCase);
+      }
+    }
+    return testCase;
+  });
+
+  testCases = tu.splitBuckets(options, testCases);
+
+  let handlers = {
+    preRun: (test) => {
+      global.currentTestConfig = undefined;
+      const configs = testCasesWithConfigs[test];
+      if (Array.isArray(configs)) {
+        if (configs.length === 0) {
+          throw "Unexpected! Have no more configs for test case " + test;
+        }
+        global.currentTestConfig = configs.shift();
+      }
+    }
+  };
+
+  return tu.performTests(options, testCases, 'chaos', tu.runInLocalArangosh, {}, handlers);
+}
+
+exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) {
+  Object.assign(allTestPaths, testPaths);
+  testFns['chaos'] = chaos;
+
+  // intentionally not turned on by default, as the suite may take a lot of time
+  // defaultFns.push('chaos');
+
+  for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; }
+  for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); }
+};

js/client/modules/@arangodb/testutils/test-utils.js

@@ -233,6 +233,9 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
           break;
         }
 
+        if (startStopHandlers && startStopHandlers.hasOwnProperty('preRun')) {
+          startStopHandlers.preRun(te);
+        }
         pu.getMemProfSnapshot(instanceInfo, options, memProfCounter++);
         print('\n' + (new Date()).toISOString() + GREEN + " [============] " + runFn.info + ': Trying', te, '...', RESET);
         let reply = runFn(options, instanceInfo, te, env);
@@ -934,7 +937,7 @@ function runInLocalArangosh (options, instanceInfo, file, addArgs) {
   } catch (ex) {
     let timeout = SetGlobalExecutionDeadlineTo(0.0);
     print(RED + 'test has thrown: ' + (timeout? "because of timeout in execution":""));
-    print(ex);
+    print(ex, ex.stack);
     print(RESET);
     return {
       timeout: timeout,

js/common/modules/@arangodb/replication-common.js

@@ -61,7 +61,8 @@ exports.reconnectRetry = function(endpoint, databaseName, user, password) {
     try {
       arango.reconnect(endpoint, databaseName, user, password);
       return;
-    } catch (ex) {
+    } catch (e) {
+      ex = e;
       print(RED + "connecting " + endpoint + " failed - retrying (" + ex.message + ")" + RESET);
     }
     sleepTime *= 2;

js/common/modules/jsunity.js

@@ -389,10 +389,21 @@ function RunTest (path, outputReply, filter) {
   var content;
   var f;
 
-  content = fs.read(path);
+  if (path.includes("test-module-")) {
+    content = `return require(${JSON.stringify(path)}).run({ runSetup, getOptions });`;
+  } else {
+    content = fs.read(path);
+  }
 
-  content = `(function(){ require('jsunity').jsUnity.attachAssertions(); return (function() { require('jsunity').setTestFilter(${JSON.stringify(filter)}); const runSetup = false; const getOptions = false; ${content} }());
-});`;
+  content = `(function(){
+    require('jsunity').jsUnity.attachAssertions();
+    return (function() {
+      require('jsunity').setTestFilter(${JSON.stringify(filter)});
+      const runSetup = false;
+      const getOptions = false;
+      ${content}
+    }());
+  });`;
   f = internal.executeScript(content, undefined, path);
 
   if (f === undefined) {

tests/js/client/communication/test-chaos-cluster.js → tests/js/client/chaos/test-chaos-load-common.inc

@@ -1,28 +1,30 @@
 /* jshint globalstrict:false, strict:false, maxlen: 200 */
 /* global fail, assertTrue, assertFalse, assertEqual,
    assertNotEqual, arango, print */
+
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License")
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Manuel Pöter
+////////////////////////////////////////////////////////////////////////////////
 
-// //////////////////////////////////////////////////////////////////////////////
-// / DISCLAIMER
-// /
-// / Copyright 2018 ArangoDB GmbH, Cologne, Germany
-// /
-// / Licensed under the Apache License, Version 2.0 (the "License")
-// / you may not use this file except in compliance with the License.
-// / You may obtain a copy of the License at
-// /
-// /     http://www.apache.org/licenses/LICENSE-2.0
-// /
-// / Unless required by applicable law or agreed to in writing, software
-// / distributed under the License is distributed on an "AS IS" BASIS,
-// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// / See the License for the specific language governing permissions and
-// / limitations under the License.
-// /
-// / Copyright holder is triAGENS GmbH, Cologne, Germany
-// /
-// / @author Manuel Pöter
-// //////////////////////////////////////////////////////////////////////////////
 'use strict';
 const _ = require('lodash');
 const jsunity = require('jsunity');
@@ -122,16 +124,15 @@ const clearAllFailurePoints = () => {
 function BaseChaosSuite(testOpts) {
   // generate a random collection name
   const cn = "UnitTests" + require("@arangodb/crypto").md5(internal.genRandomAlphaNumbers(32));
-  const communication_cn = cn + "_comm";
-
+  const coordination_cn = cn + "_comm";
 
   return {
     setUp: function () {
       db._drop(cn);
-      db._drop(communication_cn);
+      db._drop(coordination_cn);
       let rf = Math.max(2, getDBServers().length);
       db._create(cn, { numberOfShards: rf * 2, replicationFactor: rf });
-      db._create(communication_cn);
+      db._create(coordination_cn);
 
       if (testOpts.withFailurePoints) {
         let servers = getDBServers();
@@ -140,13 +141,6 @@ function BaseChaosSuite(testOpts) {
           debugSetFailAt(getEndpointById(server.id), "replicateOperations_randomize_timeout");
           debugSetFailAt(getEndpointById(server.id), "delayed_synchronous_replication_request_processing");
           debugSetFailAt(getEndpointById(server.id), "Query::setupTimeout");
-          debugSetFailAt(getEndpointById(server.id), "RocksDBCollection::insertFail1");
-          debugSetFailAt(getEndpointById(server.id), "RocksDBCollection::insertFail2");
-          debugSetFailAt(getEndpointById(server.id), "RocksDBCollection::modifyFail1");
-          debugSetFailAt(getEndpointById(server.id), "RocksDBCollection::modifyFail2");
-          debugSetFailAt(getEndpointById(server.id), "RocksDBCollection::modifyFail3");
-          debugSetFailAt(getEndpointById(server.id), "RocksDBCollection::removeFail1");
-          debugSetFailAt(getEndpointById(server.id), "RocksDBCollection::removeFail2");
         }
         servers = getCoordinators();
         assertTrue(servers.length > 0);
@@ -160,12 +154,12 @@ function BaseChaosSuite(testOpts) {
       clearAllFailurePoints();
       db._drop(cn);
 
-      const shells = db[communication_cn].all();
+      const shells = db[coordination_cn].all();
       if (shells.length > 0) {
-        print("Found remaining docs in communication collection:");
+        print("Found remaining docs in coordination collection:");
         print(shells);
       }
-      db._drop(communication_cn);
+      db._drop(coordination_cn);
     },
 
     testWorkInParallel: function () {
@@ -177,7 +171,12 @@ function BaseChaosSuite(testOpts) {
         const docs = () => {
           let result = [];
           // TODO - optionally use only single-document operations
-          const r = Math.floor(Math.random() * 2000) + 1;
+          const max = 2000;
+          const r = Math.floor(Math.random() * max) + 1;
+          if (r > (max * 0.8)) {
+            // we want ~20% of all requests to be single document operations
+            r = 1;
+          }
           for (let i = 0; i < r; ++i) {
             result.push({ _key: key() });
           }
@@ -208,12 +207,10 @@ function BaseChaosSuite(testOpts) {
 
         let query = (...args) => db._query(...args);
         let trx = null;
-        let isTransaction = false;
         if (testOpts.withStreamingTransactions && Math.random() < 0.5) {
           trx = db._createTransaction({ collections: { write: [c.name()] } });
           c = trx.collection(testOpts.collection);
           query = (...args) => trx.query(...args);
-          isTransaction = true;
         }
 
         const logAllOps = false;
@@ -222,12 +219,12 @@ function BaseChaosSuite(testOpts) {
             console.info(msg);
           }
         };
-        const ops = isTransaction ? Math.floor(Math.random() * 5) + 1 : 1;
+        const ops = trx === 0 ? 1 : Math.floor(Math.random() * 5) + 1;
         for (let op = 0; op < ops; ++op) {
           try {
             const d = Math.random();
             if (d >= 0.98 && testOpts.withTruncate) {
-              console.warn("RUNNING TRUNCATE");
+              log("RUNNING TRUNCATE");
               c.truncate();
             } else if (d >= 0.9) {
               let o = opts();
@@ -248,10 +245,12 @@ function BaseChaosSuite(testOpts) {
               let o = opts();
               let d = docs();
               log(`RUNNING INSERT WITH ${d.length} DOCS. OPTIONS: ${JSON.stringify(o)}`);
+              d = d.length == 1 ? d[0] : d;
               c.insert(d, o);
             } else {
               let d = docs();
               log(`RUNNING REMOVE WITH ${d.length} DOCS`);
+              d = d.length == 1 ? d[0] : d;
               c.remove(d);
             }
           } catch (err) {}
@@ -275,7 +274,7 @@ function BaseChaosSuite(testOpts) {
       }
 
       // run the suite for 5 minutes
-      runParallelArangoshTests(tests, 5 * 60, communication_cn);
+      runParallelArangoshTests(tests, 5 * 60, coordination_cn);
 
       print("Finished load test - waiting for cluster to get in sync before checking consistency.");
       clearAllFailurePoints();
@@ -285,44 +284,40 @@ function BaseChaosSuite(testOpts) {
   };
 }
 
-function BuildChaosSuite(opts, suffix) {
-  let suite = {};
-  deriveTestSuite(BaseChaosSuite(opts), suite, suffix);
-  return suite;
-}
-
-const params = ["IntermediateCommits", "FailurePoints", "Truncate", "VaryingOverwriteMode", "StreamingTransactions"];
+// truncate is disabled because it does not work reliably ATM
+const params = ["IntermediateCommits", "FailurePoints", /*"Truncate",*/ "VaryingOverwriteMode", "StreamingTransactions"];
 
-function addSuite(paramValues) {
+const makeConfig = (paramValues) => {
   let suffix = "";
   let opts = {};
   for (let j = 0; j < params.length; ++j) {
     suffix += paramValues[j] ? "_with_" : "_no_";
     suffix += params[j];
     opts["with" + params[j]] = paramValues[j];
-  }  
-  let func = function() { return BuildChaosSuite(opts, suffix); };
+  }
+  return { suffix: suffix, options: opts };
+}
+
+const run = () => {
+  if (!global.currentTestConfig) {
+    throw "Chaos test requires global currentTestConfig to be defined!";
+  }
+  const { options, suffix } = global.currentTestConfig;
+  print("Running chaos test with config ", options);
+
+  let func = function() {
+    let suite = {};
+    deriveTestSuite(BaseChaosSuite(options), suite, suffix);
+    return suite;
+  };
   // define the function name as it shows up as suiteName
   Object.defineProperty(func, 'name', {value: "ChaosSuite" + suffix, writable: false});
 
   jsunity.run(func);
-}
 
-/*
-This code dynamically creates test suites for all parameter combinations.
-ATM we don't use it since we don't want to include ALL suites in the PR tests, but
-we do want to have them in the nightlies, but that will be done in a separate PR.
-for (let i = 0; i < (1 << params.length); ++i) {
-  let paramValues = [];
-  for (let j = params.length - 1; j >= 0; --j) {
-    paramValues.push(Boolean(i & (1 << j)));
-  }
-  addSuite(paramValues);
+  return jsunity.done();
 }
-*/
-
-// ATM we only create a single suite with all options except Truncate, because there are still known issues.
-// Later we probably want to have all possible combinations, at least for the nightly builds.
-addSuite([true, true, false, true, true]);
 
-return jsunity.done();
+module.exports.parameters = params;
+module.exports.makeConfig = makeConfig;
+module.exports.run = run;