Bug 1141782 - Use npm-cache taskcluster docker worker image to create…

… node module cache. Use cache in gaia docker worker image. r=lightsofapollo

Makefile

@@ -703,7 +703,6 @@ endif
 NPM_INSTALLED_PROGRAMS = node_modules/.bin/mozilla-download node_modules/.bin/jshint node_modules/.bin/mocha
 $(NPM_INSTALLED_PROGRAMS): package.json node_modules
 
-
 NODE_MODULES_REV=$(shell cat gaia_node_modules.revision)
 # modules.tar and git-gaia-node-modules are the possible values for
 # $(NODE_MODULES_SRC). See the node_modules target.
@@ -727,22 +726,29 @@ git-gaia-node-modules: gaia_node_modules.revision
 	fi
 	(cd "$(NODE_MODULES_SRC)" && git fetch && git reset --hard "$(NODE_MODULES_REV)" )
 
+npm-cache:
+	@echo "Using pre-deployed cache."
+	npm rebuild marionette-js-runner
+	touch -c node_modules
+
 node_modules: gaia_node_modules.revision
 	# Running make without using a dependency ensures that we can run
 	# "make node_modules" with a custom NODE_MODULES_GIT_URL variable, and then
 	# run another target without specifying the variable
 	$(MAKE) $(NODE_MODULES_SRC)
 ifeq "$(NODE_MODULES_SRC)" "modules.tar"
 	$(TAR_WILDCARDS) --strip-components 1 -x -m -f $(NODE_MODULES_SRC) "mozilla-b2g-gaia-node-modules-*/node_modules"
-else
+else ifeq "$(NODE_MODULES_SRC)" "git-gaia-node-modules"
 	rm -fr node_modules
 	cp -R $(NODE_MODULES_SRC)/node_modules node_modules
 endif
+ifneq "$(NODE_MODULES_SRC)" "npm-cache"
 	node --version
 	npm --version
 	VIRTUALENV_EXISTS=$(VIRTUALENV_EXISTS) npm install && npm rebuild
 	@echo "node_modules installed."
 	touch -c $@
+endif
 ifeq ($(BUILDAPP),device)
 	export LANG=en_US.UTF-8;
 endif

apps/communications/contacts/test/marionette/form_test.js

@@ -29,6 +29,15 @@ marionette('Contacts > Form', function() {
     system = new System(client);
   });
 
+  teardown(function() {
+    subject = null;
+    contactsData = null;
+    selectors = null;
+    dialerSubject = null;
+    dialerSelectors = null;
+    system = null;
+  });
+
   var contactData = {
     givenName: ['Jose'],
     familyName: ['Cantera'],

apps/verticalhome/test/marionette/app_packaged_resume_test.js

@@ -13,24 +13,23 @@ marionette('Vertical Home - Packaged App Resuming Downloads', function() {
   var client = marionette.client(require(__dirname + '/client_options.js'));
 
   var server;
+  var subject;
+  var system;
+  var appInstall;
   setup(function(done) {
     var app = __dirname + '/fixtures/template_app';
     createAppServer(app, client, function(err, _server) {
       server = _server;
-      done(err);
-    });
-  });
 
-  var subject;
-  var system;
-  var appInstall;
-  setup(function() {
-    subject = client.loader.getAppClass('verticalhome');
-    system = client.loader.getAppClass('system');
-    appInstall = new AppInstall(client);
+      subject = client.loader.getAppClass('verticalhome');
+      system = client.loader.getAppClass('system');
+      appInstall = new AppInstall(client);
 
-    system.waitForStartup();
-    subject.waitForLaunch();
+      system.waitForStartup();
+      subject.waitForLaunch();
+
+      done(err);
+    });
   });
 
   teardown(function(done) {

bin/gaia-marionette

@@ -151,12 +151,7 @@ if [ "$DUMP_PATH" ]; then
   DEVICE_OPTS="$ADDITIONAL_OPTS --dump-path $DUMP_PATH"
 fi
 
-TARGET=""
-if [[ $@ == *"mocha-tbpl-reporter"* ]]; then
-  TARGET=$DIR/gaia_marionette_retry.js
-else
-  TARGET=$DIR/../node_modules/.bin/marionette-mocha
-fi
+TARGET=$DIR/../node_modules/.bin/marionette-mocha
 
 export PATH=$XPCSHELL_DIR:$PATH
 

build/docker/gaia-taskenv/Dockerfile

@@ -26,6 +26,7 @@ workdir /home/tester
 run git config --global user.email "gaia@mozilla.com"
 run git config --global user.name "gaia-taskenv"
 run npm install taskcluster-vcs@2.3.4 -g
+run npm install taskcluster-npm-cache@1.1.12 -g
 
 # run some more root commands which change frequently
 copy ./bin/entrypoint /home/tester/bin/entrypoint

build/docker/gaia-taskenv/VERSION

@@ -1 +1 @@
-0.8.0
+0.8.9

build/docker/gaia-taskenv/bin/entrypoint

@@ -37,6 +37,11 @@ step_checkout() {
   fi
 }
 
+# Set-up our npm-cache.
+step_npm_cache() {
+  taskcluster-npm-cache-get --namespace gaia.npm_cache ./package.json
+}
+
 buildbot_step 'git clone cache' step_clone
 buildbot_step 'git checkout' step_checkout
 
@@ -47,6 +52,9 @@ cd $checkout_path
 # Current revision...
 buildbot_step 'git revision...' git --no-pager log -n 1
 
+# node modules
+buildbot_step 'npm-cache-get' step_npm_cache
+
 # virtualenv setup so any python packages can be abused without root access
 buildbot_step 'virtualenv' virtualenv $HOME/.venv --system-site-packages
 source $HOME/.venv/bin/activate

build/test/integration/make_test.js

@@ -54,7 +54,9 @@ suite('Make and remake tests', function() {
     var cwd = process.cwd();
     var apps = [];
     var filter = function(file, dir) {
-      if (dir) {
+      // There are other folders we could avoid but walking the node_modules
+      // folder can be problematic when using pre-built caches (and it's slow).
+      if (dir && (file.indexOf(cwd + '/node_modules') !== 0)) {
         return true;
       }
       var pattern = new RegExp(

build/test/integration/node_modules_test.js

@@ -35,7 +35,7 @@ suite('Node modules tests [Network Required]', function() {
     rmrf('modules.tar');
     rmrf('node_modules');
     rmrf('git-gaia-node-modules');
-    helper.exec('make node_modules',
+    helper.exec('NODE_MODULES_SRC=modules.tar make node_modules',
       function(error, stdout, stderr) {
         helper.checkError(error, stdout, stderr);