diff --git a/scripts/link-core-master.js b/scripts/link-core-master.js deleted file mode 100755 index fa008b43848..00000000000 --- a/scripts/link-core-master.js +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env node -// Copyright 2020 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -// This script updates package.json to link core at master. - -const fs = require('fs'); - -let pkg = fs.readFileSync('package.json', 'utf8'); - -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-core": ".+"`, 'g'), - `"@tensorflow/tfjs-core": "link:../tfjs-core"`); - -fs.writeFileSync('package.json', pkg); diff --git a/scripts/link-master.js b/scripts/link-master.js deleted file mode 100755 index 60af2445663..00000000000 --- a/scripts/link-master.js +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env node -// Copyright 2020 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -// This script updates package.json to link following packages at master: -// tfjs-core, tfjs-layers, tfjs-converter, tfjs-data and tfjs. - -const fs = require('fs'); - -let pkg = fs.readFileSync('package.json', 'utf8'); - -// Link core. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-core": ".+"`, 'g'), - `"@tensorflow/tfjs-core": "link:../tfjs-core"`); - -// Link layers. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-layers": ".+"`, 'g'), - `"@tensorflow/tfjs-layers": "link:../tfjs-layers"`); - -// Link converter. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-converter": ".+"`, 'g'), - `"@tensorflow/tfjs-converter": "link:../tfjs-converter"`); - -// Link data. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs-data": ".+"`, 'g'), - `"@tensorflow/tfjs-data": "link:../tfjs-data"`); - -// Link union package. -pkg = `${pkg}`.replace( - new RegExp(`"@tensorflow/tfjs": ".+"`, 'g'), - `"@tensorflow/tfjs": "link:../tfjs"`); - -fs.writeFileSync('package.json', pkg); diff --git a/tfjs-converter/cloudbuild.yml b/tfjs-converter/cloudbuild.yml index df91d1fd20f..39667e4dd71 100644 --- a/tfjs-converter/cloudbuild.yml +++ b/tfjs-converter/cloudbuild.yml @@ -6,53 +6,13 @@ steps: id: 'yarn-common' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs-converter' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - -# Install converter dependencies. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['build-core'] - -# Run lint. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'lint' - args: ['lint'] - waitFor: ['yarn'] - -# Generate json file. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'gen-json' - args: ['gen-json', '--test'] - waitFor: ['yarn'] - -# Build the project. -- name: 'node:10' - dir: 'tfjs-converter' - entrypoint: 'yarn' - id: 'build' - args: ['build-ci'] - waitFor: ['gen-json'] - # Run javascript tests. - name: 'node:10' dir: 'tfjs-converter' entrypoint: 'yarn' id: 'test-js' args: ['test-ci'] - waitFor: ['build'] + waitFor: ['yarn-common'] env: ['NIGHTLY=$_NIGHTLY'] # Run python tests. @@ -60,26 +20,26 @@ steps: dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./build-pip-package.sh', '--test', '/tmp/tfjs-pips'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] - name: 'python:2' dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./run-python-tests.sh'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] - name: 'python:3.6' dir: 'tfjs-converter/python' entrypoint: 'bash' args: ['./run-python-tests.sh'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] - name: 'node:10' dir: 'tfjs-converter' entrypoint: 'yarn' id: 'test-snippets' args: ['test-snippets'] - waitFor: ['yarn'] + waitFor: ['test-js'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' diff --git a/tfjs-converter/package.json b/tfjs-converter/package.json index 44780c8905f..0640c09f3b3 100644 --- a/tfjs-converter/package.json +++ b/tfjs-converter/package.json @@ -54,16 +54,17 @@ "yalc": "~1.0.0-pre.21" }, "scripts": { - "build": "yarn link-master && yarn gen-json --test && tsc", + "build": "yarn gen-json --test && tsc", "build-ci": "yarn gen-json --test && tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", + "build-deps": "yarn build-core", + "build-deps-ci": "yarn build-core-ci", "build-npm": "./scripts/build-npm.sh", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "publish-local": "yarn build-npm && yalc push", - "test": "yarn link-master && yarn build-core && yarn && yarn gen-json --test && ts-node run_tests.ts", - "test-ci": "ts-node run_tests.ts", + "test": "yarn && yarn build-deps && yarn gen-json --test && ts-node run_tests.ts", + "test-ci": "yarn && yarn build-deps-ci && yarn build-ci && yarn lint && ts-node run_tests.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose", "make-version": "sh -c ./scripts/make-version", diff --git a/tfjs-data/cloudbuild.yml b/tfjs-data/cloudbuild.yml index f437217c500..5fff0880d83 100644 --- a/tfjs-data/cloudbuild.yml +++ b/tfjs-data/cloudbuild.yml @@ -6,53 +6,13 @@ steps: id: 'yarn-common' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs-data' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - -# Build layers from master. -- name: 'node:10' - dir: 'tfjs-data' - id: 'build-layers' - entrypoint: 'yarn' - args: ['build-layers-ci'] - waitFor: ['build-core'] - -# Install data dependencies. -- name: 'node:10' - dir: 'tfjs-data' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['build-layers'] - -# Run lint. -- name: 'node:10' - dir: 'tfjs-data' - entrypoint: 'yarn' - id: 'lint' - args: ['lint'] - waitFor: ['yarn'] - -# Build the project. -- name: 'node:10' - dir: 'tfjs-data' - entrypoint: 'yarn' - id: 'build' - args: ['build-ci'] - waitFor: ['yarn'] - # Run tests in node. - name: 'node:10' dir: 'tfjs-data' entrypoint: 'yarn' id: 'test' - args: ['test'] - waitFor: ['build'] + args: ['test-ci'] + waitFor: ['yarn-common'] # Run data snippets tests. - name: 'node:10' @@ -60,7 +20,7 @@ steps: entrypoint: 'yarn' id: 'test-snippets' args: ['test-snippets'] - waitFor: ['build'] + waitFor: ['test'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' diff --git a/tfjs-data/package.json b/tfjs-data/package.json index 23869573262..423997d708a 100644 --- a/tfjs-data/package.json +++ b/tfjs-data/package.json @@ -39,19 +39,20 @@ "yalc": "^1.0.0-pre.23" }, "scripts": { - "build": "yarn link-master && tsc", + "build": "tsc", "build-ci": "tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", "build-layers": "cd ../tfjs-layers && yarn && yarn build", "build-layers-ci": "cd ../tfjs-layers && yarn && yarn build-ci", + "build-deps": "yarn build-core && yarn build-layers", + "build-deps-ci": "yarn build-core-ci && yarn build-layers-ci", "build-npm": "./scripts/build-npm.sh", - "link-master": "../scripts/link-master.js", "link-local": "yalc link", "publish-local": "rimraf dist/ && yarn build-npm && yalc push", - "test": "yarn link-master && yarn build-core && yarn build-layers && yarn && ts-node src/test_node.ts", + "test": "yarn && yarn build-deps && ts-node src/test_node.ts", "test-browsers": "karma start --browsers='Chrome,Firefox'", - "test-ci": "ts-node src/test_node.ts", + "test-ci": "yarn && yarn build-deps-ci && yarn build-ci && yarn lint && ts-node src/test_node.ts", "test-snippets": "ts-node ./scripts/test_snippets.ts", "lint": "tslint -p . -t verbose" }, diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index 68d154ec7be..99a22fd8697 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -5,28 +5,13 @@ steps: id: 'yarn-common' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs-layers' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - -# Install layers dependencies -- name: 'node:10' - dir: 'tfjs-layers' - entrypoint: 'yarn' - id: 'yarn' - args: ['prep'] - waitFor: ['build-core'] - +# Run javascript tests - name: 'node:10' dir: 'tfjs-layers' entrypoint: 'yarn' id: 'test-browser' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY'] secretEnv: ['BROWSERSTACK_KEY'] @@ -44,6 +29,7 @@ steps: id: 'tfjs2keras-py' args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras'] waitFor: ['tfjs2keras-js'] + - name: 'node:10' dir: 'tfjs-layers' entrypoint: 'yarn' diff --git a/tfjs-layers/package.json b/tfjs-layers/package.json index d79a87c42ef..2f123e2a548 100644 --- a/tfjs-layers/package.json +++ b/tfjs-layers/package.json @@ -38,21 +38,22 @@ }, "scripts": { "prep": "yarn install && yarn build-ci", - "build": "yarn link-master && tsc", + "build": "tsc", "build-ci": "tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", + "build-deps": "yarn build-core", + "build-deps-ci": "yarn build-core-ci", "build-npm": "./scripts/build-npm.sh", "format": "./tools/clang_format_ts.sh", - "link-master": "../scripts/link-master.js", "link-local": "yalc link", "publish-local": "yarn build-npm && yalc push", - "test": "yarn link-master && yarn build-core && yarn && karma start", + "test": "yarn && yarn build-deps && karma start", + "test-ci": "./scripts/test-ci.sh", + "test-snippets": "ts-node ./scripts/test_snippets.ts", "tfjs2keras": "yarn tfjs2keras-js && yarn tfjs2keras-py --stable && yarn tfjs2keras-py --stable --tfkeras && yarn tfjs2keras-py --dev --tfkeras", "tfjs2keras-js": "./scripts/tfjs2keras-js.sh", "tfjs2keras-py": "./scripts/tfjs2keras-py.sh", - "test-ci": "./scripts/test-ci.sh", - "test-snippets": "ts-node ./scripts/test_snippets.ts", "run-browserstack": "karma start --browsers='bs_firefox_mac,bs_chrome_mac' --singleRun --reporters='dots,karma-typescript'", "lint": "tslint -p . -t verbose" }, diff --git a/tfjs-layers/scripts/test-ci.sh b/tfjs-layers/scripts/test-ci.sh index 15b222bfc81..b42d863dc9a 100755 --- a/tfjs-layers/scripts/test-ci.sh +++ b/tfjs-layers/scripts/test-ci.sh @@ -10,6 +10,8 @@ set -e # Regular testing. +yarn +yarn build-deps-ci yarn build-ci yarn lint yarn run-browserstack diff --git a/tfjs-node-gpu/cloudbuild.yml b/tfjs-node-gpu/cloudbuild.yml index 31919f3756b..2e705d3fe8f 100644 --- a/tfjs-node-gpu/cloudbuild.yml +++ b/tfjs-node-gpu/cloudbuild.yml @@ -13,21 +13,13 @@ steps: args: ['install'] waitFor: ['prep-gpu'] -# Install tfjs-node dependencies. -- name: 'node:10' - dir: 'tfjs-node-gpu' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Unit tests. - name: 'node:10' dir: 'tfjs-node-gpu' entrypoint: 'yarn' id: 'test' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] # Integration test. - name: 'node:10' @@ -35,7 +27,7 @@ steps: entrypoint: 'yarn' id: 'test-ts-integration' args: ['test-ts-integration'] - waitFor: ['yarn', 'test'] + waitFor: ['test'] # CPU / GPU package alignment. - name: 'node:10' @@ -43,7 +35,7 @@ steps: entrypoint: 'yarn' id: 'ensure-cpu-gpu-packages-align' args: ['ensure-cpu-gpu-packages-align'] - waitFor: ['yarn', 'test'] + waitFor: ['test'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' diff --git a/tfjs-node-gpu/package.json b/tfjs-node-gpu/package.json index 1abc761b444..09d9de44328 100644 --- a/tfjs-node-gpu/package.json +++ b/tfjs-node-gpu/package.json @@ -37,13 +37,12 @@ "install": "node scripts/install.js gpu download", "install-from-source": "yarn clean-deps && yarn && yarn build-addon-from-source", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "lint": "tslint -p . -t verbose", "prep": "cd node_modules/@tensorflow/tfjs-core && yarn && yarn build", "prep-gpu": "./prep-gpu.sh", "prep-gpu-windows": "./prep-gpu-windows.bat", "publish-local": "yarn prep && yalc push", - "test": "yarn link-master && yarn && yarn build-deps && ts-node src/run_tests.ts", + "test": "yarn && yarn build-deps && yarn build && ts-node src/run_tests.ts", "test-ci": "./scripts/test-ci.sh", "test-ts-integration": "./scripts/test-ts-integration.sh", "upload-windows-addon": "prep-gpu-windows.bat && ./scripts/build-and-upload-windows-addon-gpu.bat" diff --git a/tfjs-node/cloudbuild.yml b/tfjs-node/cloudbuild.yml index 1dfe558c30a..e4c872cd6b2 100644 --- a/tfjs-node/cloudbuild.yml +++ b/tfjs-node/cloudbuild.yml @@ -5,21 +5,13 @@ steps: entrypoint: 'yarn' args: ['install'] -# Install tfjs-node dependencies. -- name: 'node:10' - dir: 'tfjs-node' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['yarn-common'] - # Unit tests. - name: 'node:10' dir: 'tfjs-node' entrypoint: 'yarn' id: 'test' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] # Integration test. - name: 'node:10' @@ -27,7 +19,7 @@ steps: entrypoint: 'yarn' id: 'test-ts-integration' args: ['test-ts-integration'] - waitFor: ['yarn', 'test'] + waitFor: ['test'] # CPU / GPU package alignment. - name: 'node:10' @@ -35,7 +27,7 @@ steps: entrypoint: 'yarn' id: 'ensure-cpu-gpu-packages-align' args: ['ensure-cpu-gpu-packages-align'] - waitFor: ['yarn'] + waitFor: ['test'] timeout: 1800s logsBucket: 'gs://tfjs-build-logs' diff --git a/tfjs-node/package.json b/tfjs-node/package.json index 89263fd7fc9..5c2bb254e05 100644 --- a/tfjs-node/package.json +++ b/tfjs-node/package.json @@ -37,11 +37,10 @@ "install": "node scripts/install.js", "install-from-source": "yarn clean-deps && yarn && yarn build-addon-from-source", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "lint": "tslint -p . -t verbose", "prep": "cd node_modules/@tensorflow/tfjs-core && yarn && yarn build", "publish-local": "yarn prep && yalc push", - "test": "yarn link-master && yarn && yarn build-deps && ts-node src/run_tests.ts", + "test": "yarn && yarn build-deps && yarn build && ts-node src/run_tests.ts", "test-ci": "./scripts/test-ci.sh", "test-ts-integration": "./scripts/test-ts-integration.sh", "upload-windows-addon": "./scripts/build-and-upload-windows-addon.bat" diff --git a/tfjs-node/scripts/test-ci.sh b/tfjs-node/scripts/test-ci.sh index 51212f599af..94daf20b66a 100755 --- a/tfjs-node/scripts/test-ci.sh +++ b/tfjs-node/scripts/test-ci.sh @@ -9,6 +9,7 @@ set -e +yarn yarn build-addon-from-source yarn build-deps-ci yarn build-ci diff --git a/tfjs/cloudbuild.yml b/tfjs/cloudbuild.yml index 4e5a299a9b6..3582a7804a9 100644 --- a/tfjs/cloudbuild.yml +++ b/tfjs/cloudbuild.yml @@ -6,53 +6,13 @@ steps: entrypoint: 'yarn' args: ['install'] -# Build core from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-core' - entrypoint: 'yarn' - args: ['build-core-ci'] - waitFor: ['yarn-common'] - -# Build layers from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-layers' - entrypoint: 'yarn' - args: ['build-layers-ci'] - waitFor: ['build-core'] - -# Build converter from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-converter' - entrypoint: 'yarn' - args: ['build-converter-ci'] - waitFor: ['build-core'] - -# Build data from master. -- name: 'node:10' - dir: 'tfjs' - id: 'build-data' - entrypoint: 'yarn' - args: ['build-data-ci'] - waitFor: ['build-layers', 'build-converter'] - -# Install tfjs dependencies. -- name: 'node:10' - dir: 'tfjs' - entrypoint: 'yarn' - id: 'yarn' - args: ['install'] - waitFor: ['build-data'] - # Run tfjs tests. - name: 'node:10' dir: 'tfjs' entrypoint: 'yarn' id: 'test' args: ['test-ci'] - waitFor: ['yarn'] + waitFor: ['yarn-common'] env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY'] secretEnv: ['BROWSERSTACK_KEY', 'FIREBASE_KEY'] diff --git a/tfjs/package.json b/tfjs/package.json index b54472ef77f..3da193824bf 100644 --- a/tfjs/package.json +++ b/tfjs/package.json @@ -50,7 +50,7 @@ "yalc": "~1.0.0-pre.21" }, "scripts": { - "build": "yarn link-master && yarn build-core && yarn build-layers && yarn build-converter && yarn build-data && tsc", + "build": "tsc", "build-ci": "tsc", "build-core": "cd ../tfjs-core && yarn && yarn build", "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci", @@ -60,12 +60,13 @@ "build-converter-ci": "cd ../tfjs-converter && yarn && yarn build-ci", "build-data": "cd ../tfjs-data && yarn && yarn build", "build-data-ci": "cd ../tfjs-data && yarn && yarn build-ci", + "build-deps": "yarn build-core && yarn build-layers && yarn build-converter && yarn build-data", + "build-deps-ci": "yarn build-core-ci && yarn build-layers-ci && yarn build-converter-ci && yarn build-data-ci", "build-npm": "./scripts/build-npm.sh", "link-local": "yalc link", - "link-master": "../scripts/link-master.js", "publish-local": "yarn build-npm && yalc push", "lint": "tslint -p . -t verbose", - "test": "yarn link-master && yarn build-core && yarn build-layers && yarn build-converter && yarn build-data && yarn && ts-node ./scripts/release_notes/run_tests.ts && karma start", + "test": "yarn && yarn build-deps && yarn build && ts-node ./scripts/release_notes/run_tests.ts && karma start", "test-ci": "./scripts/test-ci.sh" }, "dependencies": { diff --git a/tfjs/scripts/test-ci.sh b/tfjs/scripts/test-ci.sh index d0fd8020bee..2caee0e7466 100755 --- a/tfjs/scripts/test-ci.sh +++ b/tfjs/scripts/test-ci.sh @@ -17,6 +17,8 @@ # Exit immediately if a command exits with a non-zero status. set -e +yarn +yarn build-deps yarn build-ci yarn lint yarn karma start --browsers='bs_firefox_mac,bs_chrome_mac' --singleRun