[WIP] Implement batching all the way from MongoDB => DDP

.circleci/config.yml

@@ -113,12 +113,25 @@ jobs:
           keys:
             - v1-dev-bundle-cache-{{ checksum "meteor" }}
             - v1-dev-bundle-cache-
+      - run:
+          name: Combine NPM Shrinkwrap Files
+          command: |
+            for d in packages/*/.npm/package; do cat $d/npm-shrinkwrap.json >> shrinkwraps.txt; done
+            for d in packages/*/.npm/plugin/*; do cat $d/npm-shrinkwrap.json >> shrinkwraps.txt; done
+      - restore_cache:
+          keys:
+            - package-npm-deps-cache-group1-v1-{{ checksum "shrinkwraps.txt" }}
+            - package-npm-deps-cache-group1-v1-
+      - restore_cache:
+          keys:
+            - package-npm-deps-cache-group2-v3-{{ checksum "shrinkwraps.txt" }}
+            - package-npm-deps-cache-group2-v3-
       - restore_cache:
           keys:
             - v2-other-deps-cache-{{ .Branch }}-{{ .Revision }}
             - v2-other-deps-cache-{{ .Branch }}-
       - restore_cache:
-          keys: 
+          keys:
             - v1-test-groups-{{ .Branch }}
             - v1-test-groups-
       - run:
@@ -166,7 +179,8 @@ jobs:
             ./meteor self-test \
               'add debugOnly and prodOnly packages' \
               --retries ${METEOR_SELF_TEST_RETRIES} \
-              --headless
+              --headless \
+              --phantom
           no_output_timeout: 20m
       - run:
           name: "Running self-test (Custom Warehouse Tests)"
@@ -176,6 +190,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --with-tag "custom-warehouse"
           no_output_timeout: 20m
       - run:
@@ -212,6 +227,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/0.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -252,6 +268,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/1.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -292,6 +309,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/2.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -332,6 +350,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/3.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -372,6 +391,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/4.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -412,6 +432,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/5.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -428,7 +449,7 @@ jobs:
           path: /tmp/core_dumps
       - store_artifacts:
           path: /tmp/memuse.txt
- 
+
   Test Group 6:
     <<: *build_machine_environment
     steps:
@@ -452,6 +473,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/6.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -492,6 +514,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/7.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -532,6 +555,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/8.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -572,6 +596,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/9.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -612,6 +637,7 @@ jobs:
               --retries ${METEOR_SELF_TEST_RETRIES} \
               --exclude "${SELF_TEST_EXCLUDE}" \
               --headless \
+              --phantom \
               --junit ./tmp/results/junit/10.xml \
               --without-tag "custom-warehouse"
           no_output_timeout: 20m
@@ -691,6 +717,52 @@ jobs:
           key: v1-dev-bundle-cache-{{ checksum "meteor" }}
           paths:
             - "dev_bundle"
+      # The package npm dependencies are split into two caches to avoid an AWS
+      # `MetadataTooLarge` error that consistently appears if we put all of
+      # these folders in the same cache
+      - save_cache:
+          key: package-npm-deps-cache-group1-v1-{{ checksum "shrinkwraps.txt" }}
+          paths:
+            - packages/meteor/.npm/package/node_modules
+            - packages/modules-runtime/.npm/package/node_modules
+            - packages/modules/.npm/package/node_modules
+            - packages/ecmascript-runtime-server/.npm/package/node_modules
+            - packages/promise/.npm/package/node_modules
+            - packages/babel-compiler/.npm/package/node_modules
+            - packages/babel-runtime/.npm/package/node_modules
+            - packages/http/.npm/package/node_modules
+            - packages/socket-stream-client/.npm/package/node_modules
+            - packages/ddp-client/.npm/package/node_modules
+            - packages/npm-mongo/.npm/package/node_modules
+            - packages/package-version-parser/.npm/package/node_modules
+            - packages/boilerplate-generator/.npm/package/node_modules
+      - save_cache:
+          key: package-npm-deps-cache-group2-v3-{{ checksum "shrinkwraps.txt" }}
+          paths:
+            - packages/xmlbuilder/.npm/package/node_modules
+            - packages/logging/.npm/package/node_modules
+            - packages/webapp/.npm/package/node_modules
+            - packages/ddp-server/.npm/package/node_modules
+            - packages/mongo/.npm/package/node_modules
+            - packages/npm-bcrypt/.npm/package/node_modules
+            - packages/email/.npm/package/node_modules
+            - packages/caching-compiler/.npm/package/node_modules
+            - packages/less/.npm/plugin/compileLessBatch/node_modules
+            - packages/non-core/blaze/packages/spacebars-compiler/.npm/package/node_modules
+            - packages/boilerplate-generator-tests/.npm/package/node_modules
+            - packages/non-core/bundle-visualizer/.npm/package/node_modules
+            - packages/d3-hierarchy/.npm/package/node_modules
+            - packages/non-core/coffeescript-compiler/.npm/package/node_modules
+            - packages/server-render/.npm/package/node_modules
+            - packages/es5-shim/.npm/package/node_modules
+            - packages/force-ssl-common/.npm/package/node_modules
+            - packages/jshint/.npm/plugin/lintJshint/node_modules
+            - packages/minifier-css/.npm/package/node_modules
+            - packages/minifier-js/.npm/package/node_modules
+            - packages/standard-minifier-css/.npm/plugin/minifyStdCSS/node_modules
+            - packages/inter-process-messaging/.npm/package/node_modules
+            - packages/fetch/.npm/package/node_modules
+            - packages/non-core/mongo-decimal/.npm/package/node_modules
       - save_cache:
           key: v2-other-deps-cache-{{ .Branch }}-{{ .Revision }}
           paths:

.travis.yml

@@ -1,11 +1,11 @@
 language: node_js
 node_js:
-  - "4.0"
+  - "8.11.1"
 cache:
   directories:
     - ".meteor"
     - ".babel-cache"
-script: TEST_PACKAGES_EXCLUDE="less" ./packages/test-in-console/run.sh
+script: TEST_PACKAGES_EXCLUDE="less" phantom=false ./packages/test-in-console/run.sh
 sudo: false
 env:
   - CXX=g++-4.8