From 8abf345f6bc21e7300bda6016163e9925f9ce166 Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 13:18:35 -0700 Subject: [PATCH 01/12] Remove link-master. Refactor package.json. --- tfjs-layers/cloudbuild.yml | 10 +--------- tfjs-layers/package.json | 8 ++++---- tfjs-layers/scripts/test-ci.sh | 1 + 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index 68d154ec7be..07d33046212 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -5,21 +5,13 @@ steps: id: 'yarn-common' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs-layers' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - # Install layers dependencies - name: 'node:10' dir: 'tfjs-layers' entrypoint: 'yarn' id: 'yarn' args: ['prep'] - waitFor: ['build-core'] + waitFor: ['yarn-common'] - name: 'node:10' dir: 'tfjs-layers' diff --git a/tfjs-layers/package.json b/tfjs-layers/package.json index d79a87c42ef..664dc3048cc 100644 --- a/tfjs-layers/package.json +++ b/tfjs-layers/package.json @@ -38,7 +38,7 @@ }, "scripts": { "prep": "yarn install && yarn build-ci", - "build": "yarn link-master && tsc", + "build": "tsc", "build-ci": "tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", @@ -47,12 +47,12 @@ "link-master": "../scripts/link-master.js", "link-local": "yalc link", "publish-local": "yarn build-npm && yalc push", - "test": "yarn link-master && yarn build-core && yarn && karma start", + "test": "yarn build-core && yarn && karma start", + "test-ci": "./scripts/test-ci.sh", + "test-snippets": "ts-node ./scripts/test_snippets.ts", "tfjs2keras": "yarn tfjs2keras-js && yarn tfjs2keras-py --stable && yarn tfjs2keras-py --stable --tfkeras && yarn tfjs2keras-py --dev --tfkeras", "tfjs2keras-js": "./scripts/tfjs2keras-js.sh", "tfjs2keras-py": "./scripts/tfjs2keras-py.sh", - "test-ci": "./scripts/test-ci.sh", - "test-snippets": "ts-node ./scripts/test_snippets.ts", "run-browserstack": "karma start --browsers='bs_firefox_mac,bs_chrome_mac' --singleRun --reporters='dots,karma-typescript'", "lint": "tslint -p . -t verbose" }, diff --git a/tfjs-layers/scripts/test-ci.sh b/tfjs-layers/scripts/test-ci.sh index 15b222bfc81..6e2242e03bf 100755 --- a/tfjs-layers/scripts/test-ci.sh +++ b/tfjs-layers/scripts/test-ci.sh @@ -10,6 +10,7 @@ set -e # Regular testing. +yarn build-core-ci yarn build-ci yarn lint yarn run-browserstack From 328bea214c76c736f2c400fa78dbf5630dbb038b Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 13:57:33 -0700 Subject: [PATCH 02/12] Refactor for layers, converter, data, union package, node, and node-gpu --- scripts/link-core-master.js | 27 ------------------- scripts/link-master.js | 49 ----------------------------------- tfjs-converter/cloudbuild.yml | 26 +------------------ tfjs-converter/package.json | 7 +++-- tfjs-data/cloudbuild.yml | 24 ----------------- tfjs-data/package.json | 9 ++++--- tfjs-layers/cloudbuild.yml | 3 ++- tfjs-layers/package.json | 3 +-- tfjs-node-gpu/package.json | 3 +-- tfjs-node/package.json | 3 +-- tfjs/cloudbuild.yml | 32 ----------------------- tfjs/package.json | 7 ++--- tfjs/scripts/test-ci.sh | 1 + 13 files changed, 19 insertions(+), 175 deletions(-) delete mode 100755 scripts/link-core-master.js delete mode 100755 scripts/link-master.js diff --git a/scripts/link-core-master.js b/scripts/link-core-master.js deleted file mode 100755 index fa008b43848..00000000000 --- a/scripts/link-core-master.js +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env node -// Copyright 2020 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -// This script updates package.json to link core at master. - -const fs = require('fs'); - -let pkg = fs.readFileSync('package.json', 'utf8'); - -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-core": ".+"`, 'g'), - `"@tensorflow/tfjs-core": "link:../tfjs-core"`); - -fs.writeFileSync('package.json', pkg); diff --git a/scripts/link-master.js b/scripts/link-master.js deleted file mode 100755 index 60af2445663..00000000000 --- a/scripts/link-master.js +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env node -// Copyright 2020 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -// This script updates package.json to link following packages at master: -// tfjs-core, tfjs-layers, tfjs-converter, tfjs-data and tfjs. - -const fs = require('fs'); - -let pkg = fs.readFileSync('package.json', 'utf8'); - -// Link core. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-core": ".+"`, 'g'), - `"@tensorflow/tfjs-core": "link:../tfjs-core"`); - -// Link layers. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-layers": ".+"`, 'g'), - `"@tensorflow/tfjs-layers": "link:../tfjs-layers"`); - -// Link converter. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-converter": ".+"`, 'g'), - `"@tensorflow/tfjs-converter": "link:../tfjs-converter"`); - -// Link data. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-data": ".+"`, 'g'), - `"@tensorflow/tfjs-data": "link:../tfjs-data"`); - -// Link union package. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs": ".+"`, 'g'), - `"@tensorflow/tfjs": "link:../tfjs"`); - -fs.writeFileSync('package.json', pkg); diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index df91d1fd20f..11aa5ecce38 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -6,21 +6,13 @@ steps: id: 'yarn-common' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs-converter' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - # Install converter dependencies. - name: 'node:10' dir: 'tfjs-converter' entrypoint: 'yarn' id: 'yarn' args: ['install'] - waitFor: ['build-core'] + waitFor: ['yarn-common'] # Run lint. - name: 'node:10' @@ -30,22 +22,6 @@ steps: args: ['lint'] waitFor: ['yarn'] -# Generate json file. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'gen-json' - args: ['gen-json', '--test'] - waitFor: ['yarn'] - -# Build the project. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'build' - args: ['build-ci'] - waitFor: ['gen-json'] - # Run javascript tests. - name: 'node:10' dir: 'tfjs-converter' diff --git a/tfjs-converter/package.json b/tfjs-converter/package.json index 44780c8905f..eca31fb3def 100644 --- a/tfjs-converter/package.json +++ b/tfjs-converter/package.json @@ -54,16 +54,15 @@ "yalc": "~1.0.0-pre.21" }, "scripts": { - "build": "yarn link-master && yarn gen-json --test && tsc", + "build": "yarn gen-json --test && tsc", "build-ci": "yarn gen-json --test && tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", "build-npm": "./scripts/build-npm.sh", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "publish-local": "yarn build-npm && yalc push", - "test": "yarn link-master && yarn build-core && yarn && yarn gen-json --test && ts-node run_tests.ts", - "test-ci": "ts-node run_tests.ts", + "test": "yarn && yarn build-core && yarn gen-json --test && ts-node run_tests.ts", + "test-ci": "yarn build-core-ci && yarn gen-json --test && yarn build-ci && ts-node run_tests.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose", "make-version": "sh -c ./scripts/make-version", diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index f437217c500..9ce6db67c9f 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -6,22 +6,6 @@ steps: id: 'yarn-common' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs-data' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - -# Build layers from master. -- name: 'node:10' - dir: 'tfjs-data' - id: 'build-layers' - entrypoint: 'yarn' - args: ['build-layers-ci'] - waitFor: ['build-core'] - # Install data dependencies. - name: 'node:10' dir: 'tfjs-data' @@ -38,14 +22,6 @@ steps: args: ['lint'] waitFor: ['yarn'] -# Build the project. -- name: 'node:10' - dir: 'tfjs-data' - entrypoint: 'yarn' - id: 'build' - args: ['build-ci'] - waitFor: ['yarn'] - # Run tests in node. - name: 'node:10' dir: 'tfjs-data' diff --git a/tfjs-data/package.json b/tfjs-data/package.json index 23869573262..67275e23a79 100644 --- a/tfjs-data/package.json +++ b/tfjs-data/package.json @@ -39,19 +39,20 @@ "yalc": "^1.0.0-pre.23" }, "scripts": { - "build": "yarn link-master && tsc", + "build": "tsc", "build-ci": "tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", "build-layers": "cd ../tfjs-layers && yarn && yarn build", "build-layers-ci": "cd ../tfjs-layers && yarn && yarn build-ci", + "build-deps": "yarn build-core && yarn build-layers", + "build-deps-ci": "yarn build-core-ci && yarn build-layers-ci", "build-npm": "./scripts/build-npm.sh", - "link-master": "../scripts/link-master.js", "link-local": "yalc link", "publish-local": "rimraf dist/ && yarn build-npm && yalc push", - "test": "yarn link-master && yarn build-core && yarn build-layers && yarn && ts-node src/test_node.ts", + "test": "yarn && yarn build-deps && ts-node src/test_node.ts", "test-browsers": "karma start --browsers='Chrome,Firefox'", - "test-ci": "ts-node src/test_node.ts", + "test-ci": "yarn build-deps-ci && yarn build-ci && ts-node src/test_node.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose" }, diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index 07d33046212..37e051db9e5 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -11,8 +11,9 @@ steps: entrypoint: 'yarn' id: 'yarn' args: ['prep'] - waitFor: ['yarn-common'] + waitFor: ['build-core'] +# Run JS tests - name: 'node:10' dir: 'tfjs-layers' entrypoint: 'yarn' diff --git a/tfjs-layers/package.json b/tfjs-layers/package.json index 664dc3048cc..2be6609c5aa 100644 --- a/tfjs-layers/package.json +++ b/tfjs-layers/package.json @@ -44,10 +44,9 @@ "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", "build-npm": "./scripts/build-npm.sh", "format": "./tools/clang_format_ts.sh", - "link-master": "../scripts/link-master.js", "link-local": "yalc link", "publish-local": "yarn build-npm && yalc push", - "test": "yarn build-core && yarn && karma start", + "test": "yarn && yarn build-core && karma start", "test-ci": "./scripts/test-ci.sh", "test-snippets": "ts-node ./scripts/test_snippets.ts", "tfjs2keras": "yarn tfjs2keras-js && yarn tfjs2keras-py --stable && yarn tfjs2keras-py --stable --tfkeras && yarn tfjs2keras-py --dev --tfkeras", diff --git a/tfjs-node-gpu/package.json b/tfjs-node-gpu/package.json index 1abc761b444..09d9de44328 100644 --- a/tfjs-node-gpu/package.json +++ b/tfjs-node-gpu/package.json @@ -37,13 +37,12 @@ "install": "node scripts/install.js gpu download", "install-from-source": "yarn clean-deps && yarn && yarn build-addon-from-source", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "lint": "tslint -p . -t verbose", "prep": "cd node_modules/@tensorflow/tfjs-core && yarn && yarn build", "prep-gpu": "./prep-gpu.sh", "prep-gpu-windows": "./prep-gpu-windows.bat", "publish-local": "yarn prep && yalc push", - "test": "yarn link-master && yarn && yarn build-deps && ts-node src/run_tests.ts", + "test": "yarn && yarn build-deps && yarn build && ts-node src/run_tests.ts", "test-ci": "./scripts/test-ci.sh", "test-ts-integration": "./scripts/test-ts-integration.sh", "upload-windows-addon": "prep-gpu-windows.bat && ./scripts/build-and-upload-windows-addon-gpu.bat" diff --git a/tfjs-node/package.json b/tfjs-node/package.json index 89263fd7fc9..5c2bb254e05 100644 --- a/tfjs-node/package.json +++ b/tfjs-node/package.json @@ -37,11 +37,10 @@ "install": "node scripts/install.js", "install-from-source": "yarn clean-deps && yarn && yarn build-addon-from-source", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "lint": "tslint -p . -t verbose", "prep": "cd node_modules/@tensorflow/tfjs-core && yarn && yarn build", "publish-local": "yarn prep && yalc push", - "test": "yarn link-master && yarn && yarn build-deps && ts-node src/run_tests.ts", + "test": "yarn && yarn build-deps && yarn build && ts-node src/run_tests.ts", "test-ci": "./scripts/test-ci.sh", "test-ts-integration": "./scripts/test-ts-integration.sh", "upload-windows-addon": "./scripts/build-and-upload-windows-addon.bat" diff --git a/tfjs/cloudbuild.yml b/tfjs/cloudbuild.yml index 4e5a299a9b6..413288250e6 100644 --- a/tfjs/cloudbuild.yml +++ b/tfjs/cloudbuild.yml @@ -6,38 +6,6 @@ steps: entrypoint: 'yarn' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - -# Build layers from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-layers' - entrypoint: 'yarn' - args: ['build-layers-ci'] - waitFor: ['build-core'] - -# Build converter from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-converter' - entrypoint: 'yarn' - args: ['build-converter-ci'] - waitFor: ['build-core'] - -# Build data from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-data' - entrypoint: 'yarn' - args: ['build-data-ci'] - waitFor: ['build-layers', 'build-converter'] - # Install tfjs dependencies. - name: 'node:10' dir: 'tfjs' diff --git a/tfjs/package.json b/tfjs/package.json index b54472ef77f..3da193824bf 100644 --- a/tfjs/package.json +++ b/tfjs/package.json @@ -50,7 +50,7 @@ "yalc": "~1.0.0-pre.21" }, "scripts": { - "build": "yarn link-master && yarn build-core && yarn build-layers && yarn build-converter && yarn build-data && tsc", + "build": "tsc", "build-ci": "tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", @@ -60,12 +60,13 @@ "build-converter-ci": "cd ../tfjs-converter && yarn && yarn build-ci", "build-data": "cd ../tfjs-data && yarn && yarn build", "build-data-ci": "cd ../tfjs-data && yarn && yarn build-ci", + "build-deps": "yarn build-core && yarn build-layers && yarn build-converter && yarn build-data", + "build-deps-ci": "yarn build-core-ci && yarn build-layers-ci && yarn build-converter-ci && yarn build-data-ci", "build-npm": "./scripts/build-npm.sh", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "publish-local": "yarn build-npm && yalc push", "lint": "tslint -p . -t verbose", - "test": "yarn link-master && yarn build-core && yarn build-layers && yarn build-converter && yarn build-data && yarn && ts-node ./scripts/release_notes/run_tests.ts && karma start", + "test": "yarn && yarn build-deps && yarn build && ts-node ./scripts/release_notes/run_tests.ts && karma start", "test-ci": "./scripts/test-ci.sh" }, "dependencies": { diff --git a/tfjs/scripts/test-ci.sh b/tfjs/scripts/test-ci.sh index d0fd8020bee..74c32318c22 100755 --- a/tfjs/scripts/test-ci.sh +++ b/tfjs/scripts/test-ci.sh @@ -17,6 +17,7 @@ # Exit immediately if a command exits with a non-zero status. set -e +yarn build-deps yarn build-ci yarn lint yarn karma start --browsers='bs_firefox_mac,bs_chrome_mac' --singleRun From a19ae2a97c2e905b96bbd41e3f436bc1f977ce70 Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 14:16:55 -0700 Subject: [PATCH 03/12] Fix cloudbuild.yml. --- tfjs-converter/cloudbuild.yml | 2 +- tfjs-data/cloudbuild.yml | 6 +++--- tfjs-layers/cloudbuild.yml | 3 ++- tfjs/cloudbuild.yml | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index 11aa5ecce38..fa2757db383 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -28,7 +28,7 @@ steps: entrypoint: 'yarn' id: 'test-js' args: ['test-ci'] - waitFor: ['build'] + waitFor: ['yarn'] env: ['NIGHTLY=$_NIGHTLY'] # Run python tests. diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index 9ce6db67c9f..8f86838fe33 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -12,7 +12,7 @@ steps: entrypoint: 'yarn' id: 'yarn' args: ['install'] - waitFor: ['build-layers'] + waitFor: ['yarn-common'] # Run lint. - name: 'node:10' @@ -28,7 +28,7 @@ steps: entrypoint: 'yarn' id: 'test' args: ['test'] - waitFor: ['build'] + waitFor: ['yarn'] # Run data snippets tests. - name: 'node:10' @@ -36,7 +36,7 @@ steps: entrypoint: 'yarn' id: 'test-snippets' args: ['test-snippets'] - waitFor: ['build'] + waitFor: ['yarn'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index 37e051db9e5..5957b1b8e9a 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -11,7 +11,7 @@ steps: entrypoint: 'yarn' id: 'yarn' args: ['prep'] - waitFor: ['build-core'] + waitFor: ['yarn-common'] # Run JS tests - name: 'node:10' @@ -37,6 +37,7 @@ steps: id: 'tfjs2keras-py' args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras'] waitFor: ['tfjs2keras-js'] + - name: 'node:10' dir: 'tfjs-layers' entrypoint: 'yarn' diff --git a/tfjs/cloudbuild.yml b/tfjs/cloudbuild.yml index 413288250e6..02bb4c9a55f 100644 --- a/tfjs/cloudbuild.yml +++ b/tfjs/cloudbuild.yml @@ -12,7 +12,7 @@ steps: entrypoint: 'yarn' id: 'yarn' args: ['install'] - waitFor: ['build-data'] + waitFor: ['yarn-common'] # Run tfjs tests. - name: 'node:10' From 83d8560138b2b3b77c434819d0702527b4a4689e Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 14:22:10 -0700 Subject: [PATCH 04/12] Fix cloudbuild.yml. --- tfjs-data/cloudbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index 8f86838fe33..c617d5034f4 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -36,7 +36,7 @@ steps: entrypoint: 'yarn' id: 'test-snippets' args: ['test-snippets'] - waitFor: ['yarn'] + waitFor: ['test'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' From e90406f62a77cb68402c3ca5794ef662df67334a Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 14:29:33 -0700 Subject: [PATCH 05/12] Fix cloudbuild.yml. --- tfjs-layers/cloudbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index 5957b1b8e9a..cc569b14126 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -10,7 +10,7 @@ steps: dir: 'tfjs-layers' entrypoint: 'yarn' id: 'yarn' - args: ['prep'] + args: ['install'] waitFor: ['yarn-common'] # Run JS tests From 8b6f89edfca8c523a11cc9fb5420cf0121fd8bc6 Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 16:33:26 -0700 Subject: [PATCH 06/12] Fix cloudbuild.yml. --- tfjs-converter/cloudbuild.yml | 14 +++++++------- tfjs-data/cloudbuild.yml | 12 ++++++------ tfjs-layers/cloudbuild.yml | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index fa2757db383..31d48b9cb15 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -14,22 +14,22 @@ steps: args: ['install'] waitFor: ['yarn-common'] -# Run lint. +# Run javascript tests. - name: 'node:10' dir: 'tfjs-converter' entrypoint: 'yarn' - id: 'lint' - args: ['lint'] + id: 'test-js' + args: ['test-ci'] waitFor: ['yarn'] + env: ['NIGHTLY=$_NIGHTLY'] -# Run javascript tests. +# Run lint. - name: 'node:10' dir: 'tfjs-converter' entrypoint: 'yarn' - id: 'test-js' - args: ['test-ci'] + id: 'lint' + args: ['lint'] waitFor: ['yarn'] - env: ['NIGHTLY=$_NIGHTLY'] # Run python tests. - name: 'gcr.io/google-appengine/python' diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index c617d5034f4..11d60b7dd16 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -14,20 +14,20 @@ steps: args: ['install'] waitFor: ['yarn-common'] -# Run lint. +# Run tests in node. - name: 'node:10' dir: 'tfjs-data' entrypoint: 'yarn' - id: 'lint' - args: ['lint'] + id: 'test' + args: ['test'] waitFor: ['yarn'] -# Run tests in node. +# Run lint. - name: 'node:10' dir: 'tfjs-data' entrypoint: 'yarn' - id: 'test' - args: ['test'] + id: 'lint' + args: ['lint'] waitFor: ['yarn'] # Run data snippets tests. diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index cc569b14126..5ed249b0af7 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -13,7 +13,7 @@ steps: args: ['install'] waitFor: ['yarn-common'] -# Run JS tests +# Run javascript tests - name: 'node:10' dir: 'tfjs-layers' entrypoint: 'yarn' From fead5010aa538235d31a8f054551e647ea8509f7 Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 16:35:09 -0700 Subject: [PATCH 07/12] Fix cloudbuild.yml. --- tfjs-converter/cloudbuild.yml | 8 -------- tfjs-converter/package.json | 2 +- tfjs-data/cloudbuild.yml | 8 -------- tfjs-data/package.json | 2 +- 4 files changed, 2 insertions(+), 18 deletions(-) diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index 31d48b9cb15..843cbf7a5dc 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -23,14 +23,6 @@ steps: waitFor: ['yarn'] env: ['NIGHTLY=$_NIGHTLY'] -# Run lint. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'lint' - args: ['lint'] - waitFor: ['yarn'] - # Run python tests. - name: 'gcr.io/google-appengine/python' dir: 'tfjs-converter/python' diff --git a/tfjs-converter/package.json b/tfjs-converter/package.json index eca31fb3def..3d250a0bdf1 100644 --- a/tfjs-converter/package.json +++ b/tfjs-converter/package.json @@ -62,7 +62,7 @@ "link-local": "yalc link", "publish-local": "yarn build-npm && yalc push", "test": "yarn && yarn build-core && yarn gen-json --test && ts-node run_tests.ts", - "test-ci": "yarn build-core-ci && yarn gen-json --test && yarn build-ci && ts-node run_tests.ts", + "test-ci": "yarn build-core-ci && yarn gen-json --test && yarn build-ci && yarn lint && ts-node run_tests.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose", "make-version": "sh -c ./scripts/make-version", diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index 11d60b7dd16..2cc465b5fce 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -22,14 +22,6 @@ steps: args: ['test'] waitFor: ['yarn'] -# Run lint. -- name: 'node:10' - dir: 'tfjs-data' - entrypoint: 'yarn' - id: 'lint' - args: ['lint'] - waitFor: ['yarn'] - # Run data snippets tests. - name: 'node:10' dir: 'tfjs-data' diff --git a/tfjs-data/package.json b/tfjs-data/package.json index 67275e23a79..9ae6df28a65 100644 --- a/tfjs-data/package.json +++ b/tfjs-data/package.json @@ -52,7 +52,7 @@ "publish-local": "rimraf dist/ && yarn build-npm && yalc push", "test": "yarn && yarn build-deps && ts-node src/test_node.ts", "test-browsers": "karma start --browsers='Chrome,Firefox'", - "test-ci": "yarn build-deps-ci && yarn build-ci && ts-node src/test_node.ts", + "test-ci": "yarn build-deps-ci && yarn build-ci && yarn lint && ts-node src/test_node.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose" }, From 89ea05250a199c9e73a541ed28cf41060eae2505 Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 18:39:45 -0700 Subject: [PATCH 08/12] Fix cloudbuild.yml. --- tfjs-converter/cloudbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index 843cbf7a5dc..709463bd05e 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -47,7 +47,7 @@ steps: entrypoint: 'yarn' id: 'test-snippets' args: ['test-snippets'] - waitFor: ['yarn'] + waitFor: ['test-js'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' From d4aa59adb3c6a7c7fbd399e61bc8e9ecf630cf42 Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 18:52:09 -0700 Subject: [PATCH 09/12] Fix cloudbuild.yml. --- tfjs-data/cloudbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index 2cc465b5fce..caeb020440a 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -19,7 +19,7 @@ steps: dir: 'tfjs-data' entrypoint: 'yarn' id: 'test' - args: ['test'] + args: ['test-ci'] waitFor: ['yarn'] # Run data snippets tests. From 650668c8e6c1ea6e5779d0a5a875d8ad6fd6db8c Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 19:40:49 -0700 Subject: [PATCH 10/12] Refactor yarn. --- tfjs-converter/cloudbuild.yml | 16 ++++------------ tfjs-converter/package.json | 2 +- tfjs-data/cloudbuild.yml | 10 +--------- tfjs-data/package.json | 2 +- tfjs-layers/cloudbuild.yml | 10 +--------- tfjs-layers/scripts/test-ci.sh | 1 + tfjs-node-gpu/cloudbuild.yml | 14 +++----------- tfjs-node/cloudbuild.yml | 14 +++----------- tfjs-node/scripts/test-ci.sh | 1 + tfjs/cloudbuild.yml | 10 +--------- tfjs/scripts/test-ci.sh | 1 + 11 files changed, 18 insertions(+), 63 deletions(-) diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index 709463bd05e..aa26a69f24f 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -6,21 +6,13 @@ steps: id: 'yarn-common' args: ['install'] -# Install converter dependencies. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Run javascript tests. - name: 'node:10' dir: 'tfjs-converter' entrypoint: 'yarn' id: 'test-js' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] env: ['NIGHTLY=$_NIGHTLY'] # Run python tests. @@ -28,19 +20,19 @@ steps: dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./build-pip-package.sh', '--test', '/tmp/tfjs-pips'] - waitFor: ['yarn'] + waitFor: ['test-js'] - name: 'python:2' dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./run-python-tests.sh'] - waitFor: ['yarn'] + waitFor: ['test-js'] - name: 'python:3.6' dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./run-python-tests.sh'] - waitFor: ['yarn'] + waitFor: ['test-js'] - name: 'node:10' dir: 'tfjs-converter' diff --git a/tfjs-converter/package.json b/tfjs-converter/package.json index 3d250a0bdf1..fee35b0149c 100644 --- a/tfjs-converter/package.json +++ b/tfjs-converter/package.json @@ -62,7 +62,7 @@ "link-local": "yalc link", "publish-local": "yarn build-npm && yalc push", "test": "yarn && yarn build-core && yarn gen-json --test && ts-node run_tests.ts", - "test-ci": "yarn build-core-ci && yarn gen-json --test && yarn build-ci && yarn lint && ts-node run_tests.ts", + "test-ci": "yarn && yarn build-core-ci && yarn gen-json --test && yarn build-ci && yarn lint && ts-node run_tests.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose", "make-version": "sh -c ./scripts/make-version", diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index caeb020440a..5fff0880d83 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -6,21 +6,13 @@ steps: id: 'yarn-common' args: ['install'] -# Install data dependencies. -- name: 'node:10' - dir: 'tfjs-data' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Run tests in node. - name: 'node:10' dir: 'tfjs-data' entrypoint: 'yarn' id: 'test' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] # Run data snippets tests. - name: 'node:10' diff --git a/tfjs-data/package.json b/tfjs-data/package.json index 9ae6df28a65..423997d708a 100644 --- a/tfjs-data/package.json +++ b/tfjs-data/package.json @@ -52,7 +52,7 @@ "publish-local": "rimraf dist/ && yarn build-npm && yalc push", "test": "yarn && yarn build-deps && ts-node src/test_node.ts", "test-browsers": "karma start --browsers='Chrome,Firefox'", - "test-ci": "yarn build-deps-ci && yarn build-ci && yarn lint && ts-node src/test_node.ts", + "test-ci": "yarn && yarn build-deps-ci && yarn build-ci && yarn lint && ts-node src/test_node.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose" }, diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index 5ed249b0af7..99a22fd8697 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -5,21 +5,13 @@ steps: id: 'yarn-common' args: ['install'] -# Install layers dependencies -- name: 'node:10' - dir: 'tfjs-layers' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Run javascript tests - name: 'node:10' dir: 'tfjs-layers' entrypoint: 'yarn' id: 'test-browser' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY'] secretEnv: ['BROWSERSTACK_KEY'] diff --git a/tfjs-layers/scripts/test-ci.sh b/tfjs-layers/scripts/test-ci.sh index 6e2242e03bf..e84c0d5f4a9 100755 --- a/tfjs-layers/scripts/test-ci.sh +++ b/tfjs-layers/scripts/test-ci.sh @@ -10,6 +10,7 @@ set -e # Regular testing. +yarn yarn build-core-ci yarn build-ci yarn lint diff --git a/tfjs-node-gpu/cloudbuild.yml b/tfjs-node-gpu/cloudbuild.yml index 31919f3756b..2e705d3fe8f 100644 --- a/tfjs-node-gpu/cloudbuild.yml +++ b/tfjs-node-gpu/cloudbuild.yml @@ -13,21 +13,13 @@ steps: args: ['install'] waitFor: ['prep-gpu'] -# Install tfjs-node dependencies. -- name: 'node:10' - dir: 'tfjs-node-gpu' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Unit tests. - name: 'node:10' dir: 'tfjs-node-gpu' entrypoint: 'yarn' id: 'test' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] # Integration test. - name: 'node:10' @@ -35,7 +27,7 @@ steps: entrypoint: 'yarn' id: 'test-ts-integration' args: ['test-ts-integration'] - waitFor: ['yarn', 'test'] + waitFor: ['test'] # CPU / GPU package alignment. - name: 'node:10' @@ -43,7 +35,7 @@ steps: entrypoint: 'yarn' id: 'ensure-cpu-gpu-packages-align' args: ['ensure-cpu-gpu-packages-align'] - waitFor: ['yarn', 'test'] + waitFor: ['test'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' diff --git a/tfjs-node/cloudbuild.yml b/tfjs-node/cloudbuild.yml index 1dfe558c30a..b081e1c8c6b 100644 --- a/tfjs-node/cloudbuild.yml +++ b/tfjs-node/cloudbuild.yml @@ -5,21 +5,13 @@ steps: entrypoint: 'yarn' args: ['install'] -# Install tfjs-node dependencies. -- name: 'node:10' - dir: 'tfjs-node' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Unit tests. - name: 'node:10' dir: 'tfjs-node' entrypoint: 'yarn' id: 'test' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] # Integration test. - name: 'node:10' @@ -27,7 +19,7 @@ steps: entrypoint: 'yarn' id: 'test-ts-integration' args: ['test-ts-integration'] - waitFor: ['yarn', 'test'] + waitFor: ['test'] # CPU / GPU package alignment. - name: 'node:10' @@ -35,7 +27,7 @@ steps: entrypoint: 'yarn' id: 'ensure-cpu-gpu-packages-align' args: ['ensure-cpu-gpu-packages-align'] - waitFor: ['yarn'] + waitFor: [test'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' diff --git a/tfjs-node/scripts/test-ci.sh b/tfjs-node/scripts/test-ci.sh index 51212f599af..94daf20b66a 100755 --- a/tfjs-node/scripts/test-ci.sh +++ b/tfjs-node/scripts/test-ci.sh @@ -9,6 +9,7 @@ set -e +yarn yarn build-addon-from-source yarn build-deps-ci yarn build-ci diff --git a/tfjs/cloudbuild.yml b/tfjs/cloudbuild.yml index 02bb4c9a55f..3582a7804a9 100644 --- a/tfjs/cloudbuild.yml +++ b/tfjs/cloudbuild.yml @@ -6,21 +6,13 @@ steps: entrypoint: 'yarn' args: ['install'] -# Install tfjs dependencies. -- name: 'node:10' - dir: 'tfjs' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Run tfjs tests. - name: 'node:10' dir: 'tfjs' entrypoint: 'yarn' id: 'test' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY'] secretEnv: ['BROWSERSTACK_KEY', 'FIREBASE_KEY'] diff --git a/tfjs/scripts/test-ci.sh b/tfjs/scripts/test-ci.sh index 74c32318c22..2caee0e7466 100755 --- a/tfjs/scripts/test-ci.sh +++ b/tfjs/scripts/test-ci.sh @@ -17,6 +17,7 @@ # Exit immediately if a command exits with a non-zero status. set -e +yarn yarn build-deps yarn build-ci yarn lint From 98d761b1ae710fe3fbd50387be2d7ea6a8ac6e37 Mon Sep 17 00:00:00 2001 From: Na Li Date: Sun, 15 Mar 2020 19:48:28 -0700 Subject: [PATCH 11/12] Fix typo. --- tfjs-node/cloudbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-node/cloudbuild.yml b/tfjs-node/cloudbuild.yml index b081e1c8c6b..e4c872cd6b2 100644 --- a/tfjs-node/cloudbuild.yml +++ b/tfjs-node/cloudbuild.yml @@ -27,7 +27,7 @@ steps: entrypoint: 'yarn' id: 'ensure-cpu-gpu-packages-align' args: ['ensure-cpu-gpu-packages-align'] - waitFor: [test'] + waitFor: ['test'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' From 2c020e4eb7f12caa327ca10648335c16b9135c84 Mon Sep 17 00:00:00 2001 From: Na Li Date: Mon, 16 Mar 2020 10:50:03 -0700 Subject: [PATCH 12/12] Use build-deps across all packages for consistency. --- tfjs-converter/cloudbuild.yml | 6 +++--- tfjs-converter/package.json | 6 ++++-- tfjs-layers/package.json | 4 +++- tfjs-layers/scripts/test-ci.sh | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index aa26a69f24f..39667e4dd71 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -20,19 +20,19 @@ steps: dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./build-pip-package.sh', '--test', '/tmp/tfjs-pips'] - waitFor: ['test-js'] + waitFor: ['yarn-common'] - name: 'python:2' dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./run-python-tests.sh'] - waitFor: ['test-js'] + waitFor: ['yarn-common'] - name: 'python:3.6' dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./run-python-tests.sh'] - waitFor: ['test-js'] + waitFor: ['yarn-common'] - name: 'node:10' dir: 'tfjs-converter' diff --git a/tfjs-converter/package.json b/tfjs-converter/package.json index fee35b0149c..0640c09f3b3 100644 --- a/tfjs-converter/package.json +++ b/tfjs-converter/package.json @@ -58,11 +58,13 @@ "build-ci": "yarn gen-json --test && tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", + "build-deps": "yarn build-core", + "build-deps-ci": "yarn build-core-ci", "build-npm": "./scripts/build-npm.sh", "link-local": "yalc link", "publish-local": "yarn build-npm && yalc push", - "test": "yarn && yarn build-core && yarn gen-json --test && ts-node run_tests.ts", - "test-ci": "yarn && yarn build-core-ci && yarn gen-json --test && yarn build-ci && yarn lint && ts-node run_tests.ts", + "test": "yarn && yarn build-deps && yarn gen-json --test && ts-node run_tests.ts", + "test-ci": "yarn && yarn build-deps-ci && yarn build-ci && yarn lint && ts-node run_tests.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose", "make-version": "sh -c ./scripts/make-version", diff --git a/tfjs-layers/package.json b/tfjs-layers/package.json index 2be6609c5aa..2f123e2a548 100644 --- a/tfjs-layers/package.json +++ b/tfjs-layers/package.json @@ -42,11 +42,13 @@ "build-ci": "tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", + "build-deps": "yarn build-core", + "build-deps-ci": "yarn build-core-ci", "build-npm": "./scripts/build-npm.sh", "format": "./tools/clang_format_ts.sh", "link-local": "yalc link", "publish-local": "yarn build-npm && yalc push", - "test": "yarn && yarn build-core && karma start", + "test": "yarn && yarn build-deps && karma start", "test-ci": "./scripts/test-ci.sh", "test-snippets": "ts-node ./scripts/test_snippets.ts", "tfjs2keras": "yarn tfjs2keras-js && yarn tfjs2keras-py --stable && yarn tfjs2keras-py --stable --tfkeras && yarn tfjs2keras-py --dev --tfkeras", diff --git a/tfjs-layers/scripts/test-ci.sh b/tfjs-layers/scripts/test-ci.sh index e84c0d5f4a9..b42d863dc9a 100755 --- a/tfjs-layers/scripts/test-ci.sh +++ b/tfjs-layers/scripts/test-ci.sh @@ -11,7 +11,7 @@ set -e # Regular testing. yarn -yarn build-core-ci +yarn build-deps-ci yarn build-ci yarn lint yarn run-browserstack