Refactor CI flow.

tfjs-backend-cpu/cloudbuild.yml

@@ -5,7 +5,7 @@ steps:
   entrypoint: 'yarn'
   args: ['install']
 
-# Install webgpu dependencies.
+# Install packages.
 - name: 'node:10'
   dir: 'tfjs-backend-cpu'
   id: 'yarn'
@@ -19,15 +19,31 @@ steps:
   id: 'build-deps'
   entrypoint: 'yarn'
   args: ['build-deps-ci']
-  waitFor: ['yarn']
+  waitFor: ['yarn-common']
+
+# Build.
+- name: 'node:10'
+  dir: 'tfjs-backend-cpu'
+  id: 'build'
+  entrypoint: 'yarn'
+  args: ['build-ci']
+  waitFor: ['yarn', 'build-deps']
+
+# Lint.
+- name: 'node:10'
+  dir: 'tfjs-backend-cpu'
+  id: 'lint'
+  entrypoint: 'yarn'
+  args: ['lint']
+  waitFor: ['yarn', 'build-deps']
 
 # Run tests.
 - name: 'node:10'
   dir: 'tfjs-backend-cpu'
   entrypoint: 'yarn'
   id: 'test-backend-cpu'
   args: ['test-ci']
-  waitFor: ['build-deps']
+  waitFor: ['yarn', 'build-deps', 'lint']
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
   secretEnv: ['BROWSERSTACK_KEY']
 

tfjs-backend-cpu/package.json

@@ -55,7 +55,7 @@
     "publish-npm": "npm publish",
     "lint": "tslint -p . -t verbose",
     "test": "yarn && yarn build-deps && ts-node run_tests.ts",
-    "test-ci": "yarn lint && yarn build-ci && ts-node run_tests.ts"
+    "test-ci": "ts-node run_tests.ts"
   },
   "dependencies": {
     "@types/seedrandom": "2.4.27",

tfjs-backend-wasm/cloudbuild.yml

@@ -5,13 +5,45 @@ steps:
   entrypoint: 'yarn'
   args: ['install']
 
-# Run tests in browser.
+# Install packages.
+- name: 'node:10'
+  dir: 'tfjs-backend-wasm'
+  entrypoint: 'yarn'
+  id: 'yarn'
+  args: ['install']
+  waitFor: ['yarn-common']
+
+# Install build-deps.
+- name: 'node:10'
+  dir: 'tfjs-backend-wasm'
+  entrypoint: 'yarn'
+  id: 'build-deps'
+  args: ['build-deps-ci']
+  waitFor: ['yarn-common']
+
+# Build.
+- name: 'node:10'
+  dir: 'tfjs-backend-wasm'
+  entrypoint: 'yarn'
+  id: 'build'
+  args: ['build-ci']
+  waitFor: ['yarn', 'build-deps']
+
+# Lint.
+- name: 'node:10'
+  dir: 'tfjs-backend-wasm'
+  entrypoint: 'yarn'
+  id: 'lint'
+  args: ['lint']
+  waitFor: ['yarn', 'build-deps']
+
+# Run JS tests.
 - name: 'node:10'
   dir: 'tfjs-backend-wasm'
   entrypoint: 'yarn'
   id: 'test-wasm'
   args: ['test-ci']
-  waitFor: ['yarn-common']
+  waitFor: ['yarn', 'build-deps', 'build', 'lint']
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
   secretEnv: ['BROWSERSTACK_KEY']
 
@@ -21,23 +53,23 @@ steps:
   entrypoint: 'yarn'
   id: 'test-cc'
   args: ['test-cc']
-  waitFor: ['test-wasm']
+  waitFor: ['build']
 
 # Check bundle size.
 - name: 'node:10'
   dir: 'tfjs-backend-wasm'
   id: 'test-bundle-size'
   entrypoint: 'yarn'
   args: ['test-bundle-size']
-  waitFor: ['test-wasm']
+  waitFor: ['build']
 
 # Lint bazel files.
 - name: 'node:10'
   dir: 'tfjs-backend-wasm'
   id: 'buildifier'
   entrypoint: 'yarn'
   args: ['buildifier-ci']
-  waitFor: ['test-wasm']
+  waitFor: ['yarn']
 
 # General configuration
 secrets:

tfjs-backend-wasm/scripts/test-ci.sh

@@ -10,9 +10,5 @@
 set -e
 
 # Regular testing.
-yarn
-yarn build-deps-ci
-yarn build-ci
-yarn lint
 yarn test-node
 yarn test-browser-ci

tfjs-backend-webgl/cloudbuild.yml

@@ -5,29 +5,45 @@ steps:
   entrypoint: 'yarn'
   args: ['install']
 
-# Install webgl dependencies.
+# Install packages.
 - name: 'node:10'
   dir: 'tfjs-backend-webgl'
   id: 'yarn'
   entrypoint: 'yarn'
   args: ['install']
   waitFor: ['yarn-common']
 
-# Build core from master.
+# Build deps.
 - name: 'node:10'
   dir: 'tfjs-backend-webgl'
-  id: 'build-core'
+  id: 'build-deps'
   entrypoint: 'yarn'
-  args: ['build-core-ci']
+  args: ['build-deps-ci']
   waitFor: ['yarn-common']
 
+# Build.
+- name: 'node:10'
+  dir: 'tfjs-backend-webgl'
+  id: 'build'
+  entrypoint: 'yarn'
+  args: ['build-ci']
+  waitFor: ['yarn', build-deps]
+
+# Lint.
+- name: 'node:10'
+  dir: 'tfjs-backend-webgl'
+  id: 'lint'
+  entrypoint: 'yarn'
+  args: ['lint']
+  waitFor: ['yarn', 'build-deps']
+
 # Run tests.
 - name: 'node:10'
   dir: 'tfjs-backend-webgl'
   entrypoint: 'yarn'
-  id: 'test-backend-webgl'
+  id: 'test'
   args: ['test-ci']
-  waitFor: ['build-core']
+  waitFor: ['yarn', 'build-deps', 'lint']
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
   secretEnv: ['BROWSERSTACK_KEY']
 

tfjs-backend-webgl/package.json

@@ -47,14 +47,16 @@
     "build": "tsc",
     "build-core": "cd ../tfjs-core && yarn && yarn build",
     "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci",
+    "build-deps": "yarn build-core",
+    "build-deps-ci": "yarn build-core-ci",
     "build-npm": "./scripts/build-npm.sh",
     "link-local": "yalc link",
     "publish-local": "rimraf dist/ && yarn build && rollup -c && yalc push",
     "publish-npm": "npm publish",
     "lint": "tslint -p . -t verbose",
-    "test": "yarn build-core && karma start",
-    "run-browserstack": "karma start --browserstack",
-    "test-ci": "./scripts/test-ci.sh"
+    "test": "yarn && yarn build-deps && karma start",
+    "test-ci": "./scripts/test-ci.sh",
+    "run-browserstack": "karma start --browserstack"
   },
   "dependencies": {
     "@types/offscreencanvas": "~2019.3.0",

tfjs-backend-webgl/scripts/test-ci.sh

@@ -16,9 +16,6 @@
 
 set -e
 
-yarn lint
-yarn build-ci
-
 if [ "$NIGHTLY" = true ]
 then
   # Run the first karma separately so it can download the BrowserStack binary

tfjs-converter/cloudbuild.yml

@@ -6,22 +6,45 @@ steps:
   id: 'yarn-common'
   args: ['install']
 
-# Install tfjs-converter dependencies
+# Install tfjs-converter dependencies.
 - name: 'node:10'
   dir: 'tfjs-converter'
   entrypoint: 'yarn'
   id: 'yarn'
   args: ['install']
   waitFor: ['yarn-common']
-  env: ['NIGHTLY=$_NIGHTLY']
+
+# Build deps.
+- name: 'node:10'
+  dir: 'tfjs-converter'
+  entrypoint: 'yarn'
+  id: 'build-deps'
+  args: ['build-deps-ci']
+  waitFor: ['yarn-common']
+
+# Build.
+- name: 'node:10'
+  dir: 'tfjs-converter'
+  entrypoint: 'yarn'
+  id: 'build'
+  args: ['build-ci']
+  waitFor: ['yarn', 'build-deps']
+
+# Lint.
+- name: 'node:10'
+  dir: 'tfjs-converter'
+  entrypoint: 'yarn'
+  id: 'lint'
+  args: ['lint']
+  waitFor: ['yarn', 'build-deps']
 
 # Run javascript tests.
 - name: 'node:10'
   dir: 'tfjs-converter'
   entrypoint: 'yarn'
   id: 'test-js'
   args: ['test-ci']
-  waitFor: ['yarn']
+  waitFor: ['yarn', 'build-deps', 'lint']
   env: ['NIGHTLY=$_NIGHTLY']
 
 # Run python tests.
@@ -48,7 +71,7 @@ steps:
   entrypoint: 'yarn'
   id: 'test-snippets'
   args: ['test-snippets']
-  waitFor: ['test-js']
+  waitFor: ['build']
 
 timeout: 1800s
 logsBucket: 'gs://tfjs-build-logs'

tfjs-converter/package.json

@@ -68,7 +68,7 @@
     "publish-local": "yarn build-npm && yalc push",
     "publish-npm": "npm publish",
     "test": "yarn && yarn build-deps && yarn gen-json --test && ts-node run_tests.ts",
-    "test-ci": "yarn && yarn build-deps-ci && yarn build-ci && yarn lint && ts-node run_tests.ts",
+    "test-ci": "ts-node run_tests.ts",
     "test-snippets": "ts-node ./scripts/test_snippets.ts",
     "lint": "tslint -p . -t verbose",
     "make-version": "sh -c ./scripts/make-version",

tfjs-core/cloudbuild.yml

@@ -13,28 +13,37 @@ steps:
   args: ['install']
   waitFor: ['yarn-common']
 
+# Build deps.
 - name: 'node:10'
   dir: 'tfjs-core'
   id: 'build-deps'
   entrypoint: 'yarn'
   args: ['build-deps-ci']
   waitFor: ['yarn']
 
-# Build
+# Build.
 - name: 'node:10'
   dir: 'tfjs-core'
   id: 'build'
   entrypoint: 'yarn'
   args: ['build-ci']
-  waitFor: ['build-deps']
+  waitFor: ['yarn', 'build-deps']
+
+# Lint.
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'lint'
+  entrypoint: 'yarn'
+  args: ['lint']
+  waitFor: ['yarn', 'build-deps']
 
 # Run unit tests.
 - name: 'node:10'
   dir: 'tfjs-core'
   id: 'test'
   entrypoint: 'yarn'
   args: ['test-ci']
-  waitFor: ['build']
+  waitFor: ['yarn', 'build-deps', 'lint']
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
   secretEnv: ['BROWSERSTACK_KEY']
 

tfjs-core/package.json

@@ -69,14 +69,14 @@
     "lint": "tslint -p . -t verbose",
     "coverage": "KARMA_COVERAGE=1 karma start --browsers='Chrome' --singleRun",
     "test": "yarn && yarn build-deps && karma start",
+    "test-ci": "./scripts/test-ci.sh",
     "test-webworker": "karma start --worker",
     "run-browserstack": "karma start --browserstack",
     "test-bundle-size": "./scripts/test-bundle-size.js",
     "test-node": "rimraf dist/ && tsc && node dist/test_node.js",
     "test-node-ci": "node dist/test_node.js",
     "test-async-backends": "rimraf dist/ && tsc && node dist/test_async_backends.js",
     "test-async-backends-ci": "node dist/test_async_backends.js",
-    "test-ci": "./scripts/test-ci.sh",
     "test-snippets": "yarn build && yarn build-cpu-backend && ts-node ./scripts/test_snippets/test_snippets.ts"
   },
   "dependencies": {

tfjs-core/scripts/test-ci.sh

@@ -16,7 +16,6 @@
 
 set -e
 
-yarn lint
 # Test in node (headless environment).
 yarn test-node-ci