Add affected packages analysis. (#2793)

Author: lina128

.gitignore

@@ -3,6 +3,7 @@
 *.tgz
 *.zip
 */diff
+*/run-ci
 **/*.pyc
 **/bundle.js
 **/*.js.map

cloudbuild.yml

@@ -5,11 +5,11 @@ steps:
   id: 'yarn'
   args: ['install']
 
-# Run diff to find modified files in each folder.
+# Run find-affected-packages to find affected files in each folder.
 - name: 'node:10'
   entrypoint: 'yarn'
-  id: 'diff'
-  args: ['diff']
+  id: 'find-affected-packages'
+  args: ['find-affected-packages']
   waitFor: ['yarn']
   env:
   - 'COMMIT_SHA=$COMMIT_SHA'
@@ -20,85 +20,85 @@ steps:
   entrypoint: 'bash'
   id: 'tfjs-core'
   args: ['./scripts/run-build.sh', 'tfjs-core']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Converter.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-converter'
   args: ['./scripts/run-build.sh', 'tfjs-converter']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Data.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-data'
   args: ['./scripts/run-build.sh', 'tfjs-data']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Layers.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-layers'
   args: ['./scripts/run-build.sh', 'tfjs-layers']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Union.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs'
   args: ['./scripts/run-build.sh', 'tfjs']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Vis.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-vis'
   args: ['./scripts/run-build.sh', 'tfjs-vis']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # WebGPU.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-backend-webgpu'
   args: ['./scripts/run-build.sh', 'tfjs-backend-webgpu']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # WASM.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-backend-wasm'
   args: ['./scripts/run-build.sh', 'tfjs-backend-wasm']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # React Native.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-react-native'
   args: ['./scripts/run-build.sh', 'tfjs-react-native']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Node CPU.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-node'
   args: ['./scripts/run-build.sh', 'tfjs-node']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Node GPU.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   id: 'tfjs-node-gpu'
   args: ['./scripts/run-build.sh', 'tfjs-node-gpu']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
 
 # Integration tests
 - name: 'node:10'
   dir: 'tfjs-core'
   id: 'test-integration'
   entrypoint: 'yarn'
   args: ['test-integration']
-  waitFor: ['diff']
+  waitFor: ['find-affected-packages']
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
   secretEnv: ['BROWSERSTACK_KEY']
 

package.json

@@ -13,7 +13,7 @@
     "typescript": "3.5.3"
   },
   "scripts": {
-    "diff": "./scripts/diff.js",
+    "find-affected-packages": "./scripts/find-affected-packages.js",
     "release": "ts-node ./scripts/release.ts",
     "release-notes": "ts-node ./scripts/release_notes/release_notes.ts",
     "test-release-notes": "ts-node ./scripts/release_notes/run_tests.ts"

scripts/diff.js renamed to scripts/find-affected-packages.js

@@ -14,15 +14,16 @@
 // limitations under the License.
 // =============================================================================
 
-const {exec} = require('./test-util');
+const {exec, constructDependencyGraph, computeAffectedPackages} =
+    require('./test-util');
 const shell = require('shelljs');
 const {readdirSync, statSync, writeFileSync} = require('fs');
 const {join} = require('path');
 const fs = require('fs');
 
 const filesWhitelistToTriggerBuild = [
   'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json',
-  'scripts/diff.js', 'scripts/run-build.sh'
+  'scripts/find-affected-packages.js', 'scripts/run-build.sh'
 ];
 
 const CLONE_PATH = 'clone';
@@ -86,7 +87,7 @@ console.log();  // Break up the console for readability.
 
 let triggeredBuilds = [];
 dirs.forEach(dir => {
-  shell.rm('-f', `${dir}/diff`);
+  shell.rm('-f', `${dir}/run-ci`);
   const diffOutput = diff(`${dir}/`);
   if (diffOutput !== '') {
     console.log(`${dir} has modified files.`);
@@ -97,13 +98,31 @@ dirs.forEach(dir => {
   const shouldDiff = diffOutput !== '' || triggerAllBuilds;
   if (shouldDiff) {
     const diffContents = whitelistDiffOutput.join('\n') + '\n' + diffOutput;
-    writeFileSync(join(dir, 'diff'), diffContents);
+    writeFileSync(join(dir, 'run-ci'), diffContents);
     triggeredBuilds.push(dir);
   }
 });
 
 console.log();  // Break up the console for readability.
 
+// Only add affected packages if not triggering all builds.
+if (!triggerAllBuilds) {
+  console.log('Computing affected packages.');
+  const affectedBuilds = new Set();
+  const dependencyGraph =
+      constructDependencyGraph('scripts/package_dependencies.json');
+  triggeredBuilds.forEach(triggeredBuild => {
+    const affectedPackages =
+        computeAffectedPackages(dependencyGraph, triggeredBuild);
+    affectedPackages.forEach(package => {
+      writeFileSync(join(package, 'run-ci'));
+      affectedBuilds.add(package);
+    });
+  });
+
+  triggeredBuilds.push(Array.from(affectedBuilds));
+}
+
 // Filter the triggered builds to log by whether a cloudbuild.yml file
 // exists for that directory.
 triggeredBuilds = triggeredBuilds.filter(

scripts/package_dependencies.json

@@ -0,0 +1,5 @@
+
+{
+  "tfjs-core": [],
+  "tfjs-converter": ["tfjs-core"]
+}

scripts/run-build.sh

@@ -17,6 +17,6 @@
 set -e
 
 DIR=$1
-if test -f "$DIR/diff"; then
+if test -f "$DIR/run-ci"; then
   gcloud builds submit . --config=$DIR/cloudbuild.yml
 fi

scripts/test-util.js

@@ -14,6 +14,7 @@
 // =============================================================================
 
 const shell = require('shelljs');
+const fs = require('fs');
 
 function exec(command, opt, ignoreCode) {
   const res = shell.exec(command, opt);
@@ -24,4 +25,57 @@ function exec(command, opt, ignoreCode) {
   return res;
 }
 
+// Construct a dependency graph keyed by dependency package.
+// Example:
+//   dependencyGraph = {
+//     "tfjs-core": ["tfjs-converter", "tfjs", ...],
+//     "tfjs": ["tfjs-node"],
+//     ...
+//   }
+function constructDependencyGraph(dependencyFilePath) {
+  const str = fs.readFileSync(dependencyFilePath, 'utf8');
+  const dependencyInfo = JSON.parse(str);
+
+  const dependencyGraph = {};
+
+  Object.keys(dependencyInfo)
+      .forEach(package => dependencyInfo[package].forEach(dependency => {
+        if (!dependencyGraph[dependency]) {
+          dependencyGraph[dependency] = [];
+        }
+        dependencyGraph[dependency].push(package);
+      }));
+
+  return dependencyGraph;
+}
+
+function computeAffectedPackages(dependencyGraph, package) {
+  const affectedPackages = new Set();
+  traverseDependencyGraph(dependencyGraph, package, affectedPackages);
+
+  return Array.from(affectedPackages);
+}
+
+// This function performs a depth-first-search to add affected packages that
+// transitively depend on the given package.
+function traverseDependencyGraph(graph, package, affectedPackages) {
+  // Terminate early if the package has been visited.
+  if (affectedPackages.has(package)) {
+    return;
+  }
+
+  const consumingPackages = graph[package];
+
+  if (!consumingPackages) {
+    return;
+  }
+
+  consumingPackages.forEach(consumingPackage => {
+    traverseDependencyGraph(graph, consumingPackage, affectedPackages);
+    affectedPackages.add(consumingPackage);
+  });
+}
+
 exports.exec = exec;
+exports.constructDependencyGraph = constructDependencyGraph;
+exports.computeAffectedPackages = computeAffectedPackages;

tfjs-layers/cloudbuild.yml

@@ -34,7 +34,7 @@ steps:
   dir: 'tfjs-layers'
   entrypoint: 'bash'
   id: 'tfjs2keras-py'
-  args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras && ./scripts/tfjs2keras-py.sh --dev --tfkeras']
+  args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras']
   waitFor: ['tfjs2keras-js']
 - name: 'node:10'
   dir: 'tfjs-layers'

tfjs-layers/scripts/tfjs2keras-py.sh

@@ -43,13 +43,13 @@ fi
 
 VENV_DIR="$(mktemp -d)_venv"
 echo "Creating virtualenv at ${VENV_DIR} ..."
-virtualenv "${VENV_DIR}"
+virtualenv -p python3 "${VENV_DIR}"
 source "${VENV_DIR}/bin/activate"
 
 if [[ "${DEV_VERSION}" == "stable" ]]; then
-  pip install -r requirements-stable.txt
+  pip3 install -r requirements-stable.txt
 else
-  pip install -r requirements-dev.txt
+  pip3 install -r requirements-dev.txt
 fi
 
 export TFJS2KERAS_TEST_USING_TF_KERAS="${TFJS2KERAS_TEST_USING_TF_KERAS}"