Refactor test runners so that individual test suites can be run one a…

…t a time Summary: In order to improve test reliability when running from the command line and especially when running on travis-ci we'd like to be able to run fewer tests at time before using phantomjs. The theory is that fewer tests will require less memory to run. At the end of each suite we restart phantomjs in order to clear the memory we've accumulated on the heap. test results: https://travis-ci.org/Khan/live-editor/builds/85978959 Test Plan: - npm test - create a pull request to check travis-ci integration Reviewers: john, pamela Reviewed By: pamela Differential Revision: https://phabricator.khanacademy.org/D22509
Khan · Oct 20, 2015 · db1381e · db1381e
1 parent d42b139
commit db1381e
Show file tree

Hide file tree

Showing 23 changed files with 197 additions and 159 deletions.
diff --git a/build/js/live-editor.output.js b/build/js/live-editor.output.js
@@ -330,7 +330,7 @@ window.LiveEditorOutput = Backbone.View.extend({
         });
 
         if (options.outputType) {
-            this.setOutput(options.outputType, true);
+            this.setOutput(options.outputType, true, options.loopProtectTimeouts);
         }
 
         // Add a timestamp property to the lintErrors and runtimeErrors arrays
@@ -357,14 +357,15 @@ window.LiveEditorOutput = Backbone.View.extend({
         window.addEventListener("message", this.handleMessage.bind(this), false);
     },
 
-    setOutput: function setOutput(outputType, enableLoopProtect) {
+    setOutput: function setOutput(outputType, enableLoopProtect, loopProtectTimeouts) {
         var OutputClass = this.outputs[outputType];
         this.output = new OutputClass({
             el: this.$el.find(".output"),
             config: this.config,
             output: this,
             type: outputType,
-            enableLoopProtect: enableLoopProtect
+            enableLoopProtect: enableLoopProtect,
+            loopProtectTimeouts: loopProtectTimeouts
         });
     },
 
@@ -426,7 +427,14 @@ window.LiveEditorOutput = Backbone.View.extend({
             if (data.enableLoopProtect != null) {
                 enableLoopProtect = data.enableLoopProtect;
             }
-            this.setOutput(outputType, enableLoopProtect);
+            var loopProtectTimeouts = {
+                initialTimeout: 2000,
+                frameTimeout: 500
+            };
+            if (data.loopProtectTimeouts != null) {
+                loopProtectTimeouts = data.loopProtectTimeouts;
+            }
+            this.setOutput(outputType, enableLoopProtect, loopProtectTimeouts);
         }
 
         // filter out debugger events

diff --git a/build/js/live-editor.output_pjs.js b/build/js/live-editor.output_pjs.js
@@ -21,6 +21,8 @@ var PJSCodeInjector = (function () {
      *      resourceCache: A ResourceCache instance.
      *      infiniteLoopCallback: A function that's when the loop protector is
      *                            triggered.
+     *      loopProtectTimeouts: An object defining initialTimeout and
+     *                           frameTimeout, see loop-protect.js for details.
      *      enabledLoopProtect: When true, loop protection code is injected.
      *      JSHint: An object containing the JSHint configuration.
      *      additionalMethods: An object containing methods that will be added
@@ -84,7 +86,7 @@ var PJSCodeInjector = (function () {
             this.initializeProps();
         }
 
-        this.loopProtector = new LoopProtector(infiniteLoopCallback, 2000, 500, true);
+        this.loopProtector = new LoopProtector(infiniteLoopCallback, options.loopProtectTimeouts, true);
 
         this.enableLoopProtect = enableLoopProtect;
 
@@ -2347,7 +2349,7 @@ window.PJSOutput = Backbone.View.extend({
         this.render();
         this.bind();
 
-        this.build(this.$canvas[0], options.enableLoopProtect);
+        this.build(this.$canvas[0], options.enableLoopProtect, options.loopProtectTimeouts);
 
         if (this.config.useDebugger && PJSDebugger) {
             iframeOverlay.createRelay(this.$canvas[0]);
@@ -2486,8 +2488,9 @@ window.PJSOutput = Backbone.View.extend({
      *
      * @param {HTMLCanvasElement} canvas
      * @param {Boolean} enableLoopProtect
+     * @param {Object} loopProtectTimeouts
      */
-    build: function build(canvas, enableLoopProtect) {
+    build: function build(canvas, enableLoopProtect, loopProtectTimeouts) {
         var _this = this;
 
         this.processing = new Processing(canvas, function (instance) {
@@ -2518,7 +2521,8 @@ window.PJSOutput = Backbone.View.extend({
             infiniteLoopCallback: this.infiniteLoopCallback.bind(this),
             enableLoopProtect: enableLoopProtect,
             JSHint: this.JSHint,
-            additionalMethods: additionalMethods
+            additionalMethods: additionalMethods,
+            loopProtectTimeouts: loopProtectTimeouts
         });
 
         this.config.runCurVersion("processing", this.processing);

diff --git a/build/js/live-editor.output_pjs_deps.js b/build/js/live-editor.output_pjs_deps.js
@@ -87404,13 +87404,15 @@ window.walkAST = function (node, path, visitors) {
  * Creates a new LoopProtector object.
  *
  * @param callback: called whenever a loop takes more than <timeout>ms to complete.
- * @param mainTimeout: threshold time used while executing main program body
- * @param asyncTimeout: treshold time used during draw() and other callbacks
+ * @param timeouts: an object containing initialTimeout and frameTimeout used
+ *                  to control how long before the loop protector is triggered
+ *                  on initial run during draw functions (or when responding to
+ *                  user events)
  * @param reportLocation: true if the location of the long running loop should be
  *                        passed to the callback. TODO(kevinb) use this for webpages
  * @constructor
  */
-window.LoopProtector = function (callback, mainTimeout, asyncTimeout, reportLocation) {
+window.LoopProtector = function (callback, timeouts, reportLocation) {
     this.callback = callback || function () {};
     this.timeout = 200;
     this.branchStartTime = 0;
@@ -87419,9 +87421,10 @@ window.LoopProtector = function (callback, mainTimeout, asyncTimeout, reportLoca
 
     this.loopBreak = esprima.parse("KAInfiniteLoopProtect()").body[0];
 
-    // cache ASTs for function calls to KAInfiniteLoopSetTimeout
-    this.mainTimeout = mainTimeout;
-    this.asyncTimeout = asyncTimeout;
+    if (timeouts) {
+        this.mainTimeout = timeouts.initialTimeout;
+        this.asyncTimeout = timeouts.frameTimeout;
+    }
 
     this.KAInfiniteLoopProtect = this._KAInfiniteLoopProtect.bind(this);
     this.KAInfiniteLoopSetTimeout = this._KAInfiniteLoopSetTimeout.bind(this);

diff --git a/build/js/live-editor.output_webpage_deps.js b/build/js/live-editor.output_webpage_deps.js
@@ -11878,13 +11878,15 @@ window.walkAST = function (node, path, visitors) {
  * Creates a new LoopProtector object.
  *
  * @param callback: called whenever a loop takes more than <timeout>ms to complete.
- * @param mainTimeout: threshold time used while executing main program body
- * @param asyncTimeout: treshold time used during draw() and other callbacks
+ * @param timeouts: an object containing initialTimeout and frameTimeout used
+ *                  to control how long before the loop protector is triggered
+ *                  on initial run during draw functions (or when responding to
+ *                  user events)
  * @param reportLocation: true if the location of the long running loop should be
  *                        passed to the callback. TODO(kevinb) use this for webpages
  * @constructor
  */
-window.LoopProtector = function (callback, mainTimeout, asyncTimeout, reportLocation) {
+window.LoopProtector = function (callback, timeouts, reportLocation) {
     this.callback = callback || function () {};
     this.timeout = 200;
     this.branchStartTime = 0;
@@ -11893,9 +11895,10 @@ window.LoopProtector = function (callback, mainTimeout, asyncTimeout, reportLoca
 
     this.loopBreak = esprima.parse("KAInfiniteLoopProtect()").body[0];
 
-    // cache ASTs for function calls to KAInfiniteLoopSetTimeout
-    this.mainTimeout = mainTimeout;
-    this.asyncTimeout = asyncTimeout;
+    if (timeouts) {
+        this.mainTimeout = timeouts.initialTimeout;
+        this.asyncTimeout = timeouts.frameTimeout;
+    }
 
     this.KAInfiniteLoopProtect = this._KAInfiniteLoopProtect.bind(this);
     this.KAInfiniteLoopSetTimeout = this._KAInfiniteLoopSetTimeout.bind(this);

diff --git a/gulpfile.js b/gulpfile.js
@@ -121,19 +121,6 @@ gulp.task("watch", function() {
         gulp.watch(paths.scripts[type], ["script_" + type]);
     });
 
-    // Run output tests when the output code changes
-    // TODO(kevinb): uncomment once running tests is rock solid
-    //gulp.watch(paths.scripts.output, ["test"]);
-    //gulp.watch(paths.scripts.output_pjs
-    //    .concat(["tests/output/pjs/*"]), ["test_output_pjs"]);
-    //gulp.watch(paths.scripts.output_webpage
-    //    .concat(["tests/output/webpage/*"]), ["test_output_webpage"]);
-    // TODO(bbondy): Uncomment when PhantomJS has support for typed arrays
-    // gulp.watch(paths.scripts.output_sql
-    //    .concat(["tests/output/sql/*"]), ["test_output_sql"]);
-    //gulp.watch(paths.scripts.tooltips
-    //    .concat(["tests/tooltips/*"]), ["test_tooltips"]);
-
     styleTypes.forEach(function(type) {
         gulp.watch(paths.styles[type], ["style_" + type]);
     });
@@ -175,15 +162,46 @@ var runTest = function(fileName) {
     };
 };
 
+// We run tests in groups so that we don't require as much memory to run them 
+// in Travis-CI.
+var pjs_tests = ["jshint", "output", "assert", "ast_transform"];
+
+pjs_tests.forEach(function(test) {
+    gulp.task("test_output_pjs_" + test, ["script_output_pjs"], function() {
+        return gulp.src("tests/output/pjs/index.html")
+            .pipe(mochaRunner({ test: test + "_test.js" }));
+    });
+});
+
+gulp.task("test_output_pjs", function(callback) {
+    var sequence = pjs_tests.map(function(test) {
+        return "test_output_pjs_" + test;
+    });
+    sequence.push(callback);
+    runSequence.apply(null, sequence);
+});
 
-gulp.task("test_output_pjs", ["script_output_pjs"], function() {
+// TODO(kevinb) add this to test_output_pjs after adding try/catch to event handlers
+gulp.task("test_output_pjs_async", ["script_output_pjs"], function() {
     return gulp.src("tests/output/pjs/index.html")
-        .pipe(mochaRunner());
+        .pipe(mochaRunner({ test: "async_test.js" }));
 });
 
-gulp.task("test_output_webpage", ["script_output_webpage"], function() {
-    return gulp.src("tests/output/webpage/ci-index.html")
-        .pipe(mochaRunner());
+var webpage_tests = ["assert", "output", "transform"];
+
+webpage_tests.forEach(function(test) {
+    gulp.task("test_output_webpage_" + test, ["script_output_pjs"], function() {
+        return gulp.src("tests/output/webpage/index.html")
+            .pipe(mochaRunner({ test: test + "_test.js" }));
+    });
+});
+
+gulp.task("test_output_webpage", ["script_output_webpage"], function(callback) {
+    var sequence = webpage_tests.map(function(test) {
+        return "test_output_webpage_" + test;
+    });
+    sequence.push(callback);
+    runSequence.apply(null, sequence);
 });
 
 // TODO(kevinb): Uncomment when phantomJS has been upgraded to 2.0

diff --git a/js/output/pjs/pjs-code-injector.js b/js/output/pjs/pjs-code-injector.js
@@ -14,6 +14,8 @@ class PJSCodeInjector {
      *      resourceCache: A ResourceCache instance.
      *      infiniteLoopCallback: A function that's when the loop protector is
      *                            triggered.
+     *      loopProtectTimeouts: An object defining initialTimeout and
+     *                           frameTimeout, see loop-protect.js for details.
      *      enabledLoopProtect: When true, loop protection code is injected.
      *      JSHint: An object containing the JSHint configuration.
      *      additionalMethods: An object containing methods that will be added
@@ -81,7 +83,7 @@ class PJSCodeInjector {
         }
 
         this.loopProtector = new LoopProtector(
-            infiniteLoopCallback, 2000, 500, true);
+            infiniteLoopCallback, options.loopProtectTimeouts, true);
 
         this.enableLoopProtect = enableLoopProtect;
 

diff --git a/js/output/pjs/pjs-output.js b/js/output/pjs/pjs-output.js
@@ -23,7 +23,10 @@ window.PJSOutput = Backbone.View.extend({
         this.render();
         this.bind();
 
-        this.build(this.$canvas[0], options.enableLoopProtect);
+        this.build(
+            this.$canvas[0],
+            options.enableLoopProtect,
+            options.loopProtectTimeouts);
 
         if (this.config.useDebugger && PJSDebugger) {
             iframeOverlay.createRelay(this.$canvas[0]);
@@ -175,8 +178,9 @@ window.PJSOutput = Backbone.View.extend({
      *
      * @param {HTMLCanvasElement} canvas
      * @param {Boolean} enableLoopProtect
+     * @param {Object} loopProtectTimeouts
      */
-    build: function(canvas, enableLoopProtect) {
+    build: function(canvas, enableLoopProtect, loopProtectTimeouts) {
         this.processing = new Processing(canvas, (instance) => {
             instance.draw = this.DUMMY;
         });
@@ -205,7 +209,8 @@ window.PJSOutput = Backbone.View.extend({
             infiniteLoopCallback: this.infiniteLoopCallback.bind(this),
             enableLoopProtect: enableLoopProtect,
             JSHint: this.JSHint,
-            additionalMethods: additionalMethods
+            additionalMethods: additionalMethods,
+            loopProtectTimeouts: loopProtectTimeouts,
         });
 
         this.config.runCurVersion("processing", this.processing);

diff --git a/js/output/shared/loop-protect.js b/js/output/shared/loop-protect.js
@@ -2,13 +2,15 @@
  * Creates a new LoopProtector object.
  *
  * @param callback: called whenever a loop takes more than <timeout>ms to complete.
- * @param mainTimeout: threshold time used while executing main program body
- * @param asyncTimeout: treshold time used during draw() and other callbacks
+ * @param timeouts: an object containing initialTimeout and frameTimeout used
+ *                  to control how long before the loop protector is triggered
+ *                  on initial run during draw functions (or when responding to
+ *                  user events)
  * @param reportLocation: true if the location of the long running loop should be
  *                        passed to the callback. TODO(kevinb) use this for webpages
  * @constructor
  */
-window.LoopProtector = function(callback, mainTimeout, asyncTimeout, reportLocation) {
+window.LoopProtector = function(callback, timeouts, reportLocation) {
     this.callback = callback || function () { };
     this.timeout = 200;
     this.branchStartTime = 0;
@@ -17,9 +19,10 @@ window.LoopProtector = function(callback, mainTimeout, asyncTimeout, reportLocat
 
     this.loopBreak = esprima.parse("KAInfiniteLoopProtect()").body[0];
 
-    // cache ASTs for function calls to KAInfiniteLoopSetTimeout
-    this.mainTimeout = mainTimeout;
-    this.asyncTimeout = asyncTimeout;
+    if (timeouts) {
+        this.mainTimeout = timeouts.initialTimeout;
+        this.asyncTimeout = timeouts.frameTimeout;
+    }
 
     this.KAInfiniteLoopProtect = this._KAInfiniteLoopProtect.bind(this);
     this.KAInfiniteLoopSetTimeout = this._KAInfiniteLoopSetTimeout.bind(this);

diff --git a/js/output/shared/output.js b/js/output/shared/output.js
@@ -16,7 +16,7 @@ window.LiveEditorOutput = Backbone.View.extend({
         });
 
         if (options.outputType) {
-            this.setOutput(options.outputType, true);
+            this.setOutput(options.outputType, true, options.loopProtectTimeouts);
         }
 
         // Add a timestamp property to the lintErrors and runtimeErrors arrays
@@ -44,14 +44,15 @@ window.LiveEditorOutput = Backbone.View.extend({
             this.handleMessage.bind(this), false);
     },
 
-    setOutput: function(outputType, enableLoopProtect) {
+    setOutput: function(outputType, enableLoopProtect, loopProtectTimeouts) {
         var OutputClass = this.outputs[outputType];
         this.output = new OutputClass({
             el: this.$el.find(".output"),
             config: this.config,
             output: this,
             type: outputType,
-            enableLoopProtect: enableLoopProtect
+            enableLoopProtect: enableLoopProtect,
+            loopProtectTimeouts: loopProtectTimeouts
         });
     },
 
@@ -113,7 +114,14 @@ window.LiveEditorOutput = Backbone.View.extend({
             if (data.enableLoopProtect != null) {
                 enableLoopProtect = data.enableLoopProtect;
             }
-            this.setOutput(outputType, enableLoopProtect);
+            var loopProtectTimeouts = {
+                initialTimeout: 2000,
+                frameTimeout: 500
+            };
+            if (data.loopProtectTimeouts != null) {
+                loopProtectTimeouts = data.loopProtectTimeouts;
+            }
+            this.setOutput(outputType, enableLoopProtect, loopProtectTimeouts);
         }
 
         // filter out debugger events

diff --git a/lint_blacklist.txt b/lint_blacklist.txt
@@ -4,6 +4,7 @@ external/
 tests/
 bower_components/
 node_modules/
+testutil/
 js/output/
 gulpfile.js
 

diff --git a/package.json b/package.json
@@ -5,7 +5,7 @@
   "main": "dist/live-editor.js",
   "scripts": {
     "build": "gulp build",
-    "test": "gulp test_output_webpage; gulp test_tooltips",
+    "test": "gulp test",
     "install": "rm -rf node_modules/gulp-handlebars/node_modules/handlebars"
   },
   "repository": {

diff --git a/tests/output/pjs/ast_transform_tests.js → tests/output/pjs/ast_transform_test.js b/tests/output/pjs/ast_transform_tests.js → tests/output/pjs/ast_transform_test.js