From 913147fd458a50cc8327bf52dbea8ab89746e6b5 Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:04:01 -0800 Subject: [PATCH 01/26] Add affected packages analysis. --- scripts/dependency.json | 12 +++++++++ scripts/diff.js | 16 ++++++++++++ scripts/test-util.js | 55 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 scripts/dependency.json diff --git a/scripts/dependency.json b/scripts/dependency.json new file mode 100644 index 00000000000..464c1f08c93 --- /dev/null +++ b/scripts/dependency.json @@ -0,0 +1,12 @@ +{ + "packages": [ + { + "package": "tfjs-core", + "dependencies": [] + }, + { + "package": "tfjs-converter", + "dependencies": ["tfjs-core"] + } + ] +} diff --git a/scripts/diff.js b/scripts/diff.js index 9e000e66feb..d78b431bbd3 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -104,6 +104,22 @@ dirs.forEach(dir => { console.log(); // Break up the console for readability. +let dependencyGraph = constructDependencyGraph('dependency.json'); + +triggeredBuilds.forEach(triggeredBuild => { + const affectedPackages = + calculateAffectedPackages(dependencyGraph, triggeredBuild); + if (affectedPackages.length > 0) { + affectedPackages.forEach(package => { + writeFileSync(join(package, 'diff'), ''); + triggeredBuild.push(package); + }) + } +}); + +// Deduplicate the triggered builds for log. +triggeredBuilds = [...new Set(triggeredBuilds)]; + // Filter the triggered builds to log by whether a cloudbuild.yml file // exists for that directory. triggeredBuilds = triggeredBuilds.filter( diff --git a/scripts/test-util.js b/scripts/test-util.js index c03465ce158..66bd486ea95 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -14,6 +14,7 @@ // ============================================================================= const shell = require('shelljs'); +const fs = require('fs'); function exec(command, opt, ignoreCode) { const res = shell.exec(command, opt); @@ -24,4 +25,58 @@ function exec(command, opt, ignoreCode) { return res; } +function calculateAffectedPackages(dependencyGraph, package) { + const affectedPackages = new Set(); + traverseDependencyGraph(graph, package, affectedPackages); + + return affectedPackages; +} + +function constructDependencyGraph(path) { + try { + const str = fs.readFileSync(path); + } catch (err) { + console.log('Error reading dependency file.'); + return {}; + } + + try { + const dependencyInfo = JSON.parse(str); + } catch (err) { + console.log('Error parsing dependency file to JSON.'); + return {}; + } + + const dependencyGraph = {}; + + dependencyInfo.packages.forEach( + package => {package.dependencies.forEach(dependency => { + if (!dependencyGraph[dependency]) { + dependencyGraph[dependency] = []; + } + dependencyGraph[dependency].push(package.package); + })}); + + return dependencyGraph; +} + +function traverseDependencyGraph(graph, package, affectedPackages) { + if (affectedPackages.has(package)) { + return; + } + + const consumingPackages = graph[package]; + + if (!consumingPackages) { + return; + } + + consumingPackages.forEach(consumingPackage => { + traverseDependencyGraph(graph, consumingPackage, affectedPackages); + affectedPackages.add(consumingPackage); + }); +} + exports.exec = exec; +exports.constructDependencyGraph = constructDependencyGraph; +exports.calculateAffectedPackages = calculateAffectedPackages; From 42905fe9c24c717cac22c6c8bb33e8a77534f4c8 Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:13:53 -0800 Subject: [PATCH 02/26] Add run condition. --- scripts/diff.js | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/scripts/diff.js b/scripts/diff.js index d78b431bbd3..676868ce190 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -14,7 +14,8 @@ // limitations under the License. // ============================================================================= -const {exec} = require('./test-util'); +const {exec, constructDependencyGraph, calculateAffectedPackages} = + require('./test-util'); const shell = require('shelljs'); const {readdirSync, statSync, writeFileSync} = require('fs'); const {join} = require('path'); @@ -104,21 +105,23 @@ dirs.forEach(dir => { console.log(); // Break up the console for readability. -let dependencyGraph = constructDependencyGraph('dependency.json'); - -triggeredBuilds.forEach(triggeredBuild => { - const affectedPackages = - calculateAffectedPackages(dependencyGraph, triggeredBuild); - if (affectedPackages.length > 0) { - affectedPackages.forEach(package => { - writeFileSync(join(package, 'diff'), ''); - triggeredBuild.push(package); - }) - } -}); - -// Deduplicate the triggered builds for log. -triggeredBuilds = [...new Set(triggeredBuilds)]; +if (!triggerAllBuilds) { + let dependencyGraph = constructDependencyGraph('dependency.json'); + + triggeredBuilds.forEach(triggeredBuild => { + const affectedPackages = + calculateAffectedPackages(dependencyGraph, triggeredBuild); + if (affectedPackages.length > 0) { + affectedPackages.forEach(package => { + writeFileSync(join(package, 'diff'), ''); + triggeredBuild.push(package); + }) + } + }); + + // Deduplicate the triggered builds for log. + triggeredBuilds = [...new Set(triggeredBuilds)]; +} // Filter the triggered builds to log by whether a cloudbuild.yml file // exists for that directory. From 400b8e62a2204131f4bc659dcff5512fecfd6257 Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:17:07 -0800 Subject: [PATCH 03/26] Remove diff from whitelist for testing. --- scripts/diff.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/diff.js b/scripts/diff.js index 676868ce190..9fa54075783 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -23,7 +23,7 @@ const fs = require('fs'); const filesWhitelistToTriggerBuild = [ 'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json', - 'scripts/diff.js', 'scripts/run-build.sh' + 'scripts/run-build.sh' ]; const CLONE_PATH = 'clone'; From 5c77e5655d873a2136620c417bad57ed8d7cb0c8 Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:20:05 -0800 Subject: [PATCH 04/26] . --- scripts/test-util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-util.js b/scripts/test-util.js index 66bd486ea95..093e96ea1d4 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -27,7 +27,7 @@ function exec(command, opt, ignoreCode) { function calculateAffectedPackages(dependencyGraph, package) { const affectedPackages = new Set(); - traverseDependencyGraph(graph, package, affectedPackages); + traverseDependencyGraph(dependencyGraph, package, affectedPackages); return affectedPackages; } From 981a2ed288859cc1ebbd3cfd6699b010ed017330 Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:29:45 -0800 Subject: [PATCH 05/26] Change file path. --- scripts/diff.js | 2 +- scripts/test-util.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/diff.js b/scripts/diff.js index 9fa54075783..38a8b1b0d26 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -106,7 +106,7 @@ dirs.forEach(dir => { console.log(); // Break up the console for readability. if (!triggerAllBuilds) { - let dependencyGraph = constructDependencyGraph('dependency.json'); + let dependencyGraph = constructDependencyGraph('scripts/dependency.json'); triggeredBuilds.forEach(triggeredBuild => { const affectedPackages = diff --git a/scripts/test-util.js b/scripts/test-util.js index 093e96ea1d4..131c8409165 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -34,7 +34,7 @@ function calculateAffectedPackages(dependencyGraph, package) { function constructDependencyGraph(path) { try { - const str = fs.readFileSync(path); + const str = fs.readFileSync(path, 'utf8'); } catch (err) { console.log('Error reading dependency file.'); return {}; From 1d3af68090e18254c37c591d6d6828c827b5fe04 Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:50:00 -0800 Subject: [PATCH 06/26] debug. --- scripts/test-util.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test-util.js b/scripts/test-util.js index 131c8409165..f04c2d4853a 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -41,8 +41,10 @@ function constructDependencyGraph(path) { } try { + console.log(str); const dependencyInfo = JSON.parse(str); } catch (err) { + console.log(err); console.log('Error parsing dependency file to JSON.'); return {}; } From 9aee28d067bc7ef1ac26132779359cb9656fc34a Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:52:35 -0800 Subject: [PATCH 07/26] . --- scripts/test-util.js | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/scripts/test-util.js b/scripts/test-util.js index f04c2d4853a..01d251d0ced 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -33,21 +33,8 @@ function calculateAffectedPackages(dependencyGraph, package) { } function constructDependencyGraph(path) { - try { - const str = fs.readFileSync(path, 'utf8'); - } catch (err) { - console.log('Error reading dependency file.'); - return {}; - } - - try { - console.log(str); - const dependencyInfo = JSON.parse(str); - } catch (err) { - console.log(err); - console.log('Error parsing dependency file to JSON.'); - return {}; - } + const str = fs.readFileSync(path, 'utf8'); + const dependencyInfo = JSON.parse(str); const dependencyGraph = {}; From b106dc2a1d1cda9f0ebcaf46ee79444eee46a59c Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:54:58 -0800 Subject: [PATCH 08/26] Test Core change. --- tfjs-core/src/backends/non_max_suppression_impl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-core/src/backends/non_max_suppression_impl.ts b/tfjs-core/src/backends/non_max_suppression_impl.ts index 3f7bb8d8961..351f53e4451 100644 --- a/tfjs-core/src/backends/non_max_suppression_impl.ts +++ b/tfjs-core/src/backends/non_max_suppression_impl.ts @@ -48,7 +48,7 @@ export function nonMaxSuppressionV5( iouThreshold: number, scoreThreshold: number, softNmsSigma: number): NamedTensorMap { // For NonMaxSuppressionV5Op, we always return a second output holding - // corresponding scores. + // corresponding scores const returnScoresTensor = true; const result = nonMaxSuppressionImpl_( From 5cc09234c8e2d3962f05b8be17acf03e2cb68936 Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 18:58:07 -0800 Subject: [PATCH 09/26] Debug. --- scripts/diff.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/diff.js b/scripts/diff.js index 38a8b1b0d26..4f55da9d7f5 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -107,10 +107,11 @@ console.log(); // Break up the console for readability. if (!triggerAllBuilds) { let dependencyGraph = constructDependencyGraph('scripts/dependency.json'); - + console.log(dependencyGraph); triggeredBuilds.forEach(triggeredBuild => { const affectedPackages = calculateAffectedPackages(dependencyGraph, triggeredBuild); + console.log(affectedPackages); if (affectedPackages.length > 0) { affectedPackages.forEach(package => { writeFileSync(join(package, 'diff'), ''); @@ -121,6 +122,7 @@ if (!triggerAllBuilds) { // Deduplicate the triggered builds for log. triggeredBuilds = [...new Set(triggeredBuilds)]; + console.log(triggeredBuilds); } // Filter the triggered builds to log by whether a cloudbuild.yml file From 5eff90994d0df244b247f7492066c68131a6bfeb Mon Sep 17 00:00:00 2001 From: Na Li Date: Tue, 25 Feb 2020 19:01:19 -0800 Subject: [PATCH 10/26] . --- scripts/test-util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-util.js b/scripts/test-util.js index 01d251d0ced..e68432e6323 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -29,7 +29,7 @@ function calculateAffectedPackages(dependencyGraph, package) { const affectedPackages = new Set(); traverseDependencyGraph(dependencyGraph, package, affectedPackages); - return affectedPackages; + return [...affectedPackages]; } function constructDependencyGraph(path) { From 6d23908cc1b9cb556c313b6ac237106db3ab31e5 Mon Sep 17 00:00:00 2001 From: Na Li Date: Wed, 26 Feb 2020 16:27:40 -0800 Subject: [PATCH 11/26] debug. --- scripts/diff.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/diff.js b/scripts/diff.js index 4f55da9d7f5..af304bfde2a 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -111,11 +111,12 @@ if (!triggerAllBuilds) { triggeredBuilds.forEach(triggeredBuild => { const affectedPackages = calculateAffectedPackages(dependencyGraph, triggeredBuild); - console.log(affectedPackages); + console.log('affectedPackages are: ', affectedPackages); if (affectedPackages.length > 0) { affectedPackages.forEach(package => { writeFileSync(join(package, 'diff'), ''); triggeredBuild.push(package); + console.log('adding to triggeredBuild', package); }) } }); From 7d190c5d84328425d5da7b43d5f821e9f50aaa69 Mon Sep 17 00:00:00 2001 From: Na Li Date: Wed, 26 Feb 2020 16:30:57 -0800 Subject: [PATCH 12/26] debug. --- scripts/diff.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/diff.js b/scripts/diff.js index af304bfde2a..8b29f9e1d4c 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -115,7 +115,7 @@ if (!triggerAllBuilds) { if (affectedPackages.length > 0) { affectedPackages.forEach(package => { writeFileSync(join(package, 'diff'), ''); - triggeredBuild.push(package); + triggeredBuilds.push(package); console.log('adding to triggeredBuild', package); }) } From c60d5abe0f49baa4c52f049f83cb6561ab8303df Mon Sep 17 00:00:00 2001 From: Na Li Date: Wed, 26 Feb 2020 16:35:24 -0800 Subject: [PATCH 13/26] debug. --- scripts/dependency.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/dependency.json b/scripts/dependency.json index 464c1f08c93..b7851a2ef14 100644 --- a/scripts/dependency.json +++ b/scripts/dependency.json @@ -7,6 +7,22 @@ { "package": "tfjs-converter", "dependencies": ["tfjs-core"] + }, + { + "package": "tfjs-layers", + "dependencies": ["tfjs-core"] + }, + { + "package": "tfjs-data", + "dependencies": ["tfjs-core", "tfjs-layers"] + }, + { + "package": "tfjs", + "dependencies": ["tfjs-core", "tfjs-converter", "tfjs-layers", "tfjs-data"] + }, + { + "package": "tfjs-node", + "dependencies": ["tfjs"] } ] } From 6459d60c1efda796d70a9673409dd71b574afdd7 Mon Sep 17 00:00:00 2001 From: Na Li Date: Wed, 26 Feb 2020 17:19:44 -0800 Subject: [PATCH 14/26] Clean up. --- scripts/dependency.json | 16 ----------- scripts/diff.js | 17 ++++------- scripts/test-util.js | 28 +++++++++++++------ .../src/backends/non_max_suppression_impl.ts | 2 +- 4 files changed, 26 insertions(+), 37 deletions(-) diff --git a/scripts/dependency.json b/scripts/dependency.json index b7851a2ef14..464c1f08c93 100644 --- a/scripts/dependency.json +++ b/scripts/dependency.json @@ -7,22 +7,6 @@ { "package": "tfjs-converter", "dependencies": ["tfjs-core"] - }, - { - "package": "tfjs-layers", - "dependencies": ["tfjs-core"] - }, - { - "package": "tfjs-data", - "dependencies": ["tfjs-core", "tfjs-layers"] - }, - { - "package": "tfjs", - "dependencies": ["tfjs-core", "tfjs-converter", "tfjs-layers", "tfjs-data"] - }, - { - "package": "tfjs-node", - "dependencies": ["tfjs"] } ] } diff --git a/scripts/diff.js b/scripts/diff.js index 8b29f9e1d4c..a8c1b410ba6 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -23,7 +23,7 @@ const fs = require('fs'); const filesWhitelistToTriggerBuild = [ 'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json', - 'scripts/run-build.sh' + 'scripts/diff.js', 'scripts/run-build.sh' ]; const CLONE_PATH = 'clone'; @@ -105,25 +105,20 @@ dirs.forEach(dir => { console.log(); // Break up the console for readability. +// Only add affected packages if not triggering all builds. if (!triggerAllBuilds) { let dependencyGraph = constructDependencyGraph('scripts/dependency.json'); - console.log(dependencyGraph); triggeredBuilds.forEach(triggeredBuild => { const affectedPackages = calculateAffectedPackages(dependencyGraph, triggeredBuild); - console.log('affectedPackages are: ', affectedPackages); - if (affectedPackages.length > 0) { - affectedPackages.forEach(package => { - writeFileSync(join(package, 'diff'), ''); - triggeredBuilds.push(package); - console.log('adding to triggeredBuild', package); - }) - } + affectedPackages.forEach(package => { + writeFileSync(join(package, 'diff'), ''); + triggeredBuilds.push(package); + }); }); // Deduplicate the triggered builds for log. triggeredBuilds = [...new Set(triggeredBuilds)]; - console.log(triggeredBuilds); } // Filter the triggered builds to log by whether a cloudbuild.yml file diff --git a/scripts/test-util.js b/scripts/test-util.js index e68432e6323..89e18650939 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -25,15 +25,15 @@ function exec(command, opt, ignoreCode) { return res; } -function calculateAffectedPackages(dependencyGraph, package) { - const affectedPackages = new Set(); - traverseDependencyGraph(dependencyGraph, package, affectedPackages); - - return [...affectedPackages]; -} - -function constructDependencyGraph(path) { - const str = fs.readFileSync(path, 'utf8'); +// Construct a dependency graph keyed by dependency package. +// Example: +// dependencyGraph = { +// "tfjs-core": ["tfjs-converter", "tfjs", ...], +// "tfjs": ["tfjs-node"], +// ... +// } +function constructDependencyGraph(dependencyFilePath) { + const str = fs.readFileSync(dependencyFilePath, 'utf8'); const dependencyInfo = JSON.parse(str); const dependencyGraph = {}; @@ -49,7 +49,17 @@ function constructDependencyGraph(path) { return dependencyGraph; } +function calculateAffectedPackages(dependencyGraph, package) { + const affectedPackages = new Set(); + traverseDependencyGraph(dependencyGraph, package, affectedPackages); + + return [...affectedPackages]; +} + +// This function performs a depth-first-search to add affected packages that +// transitively depend on the given package. function traverseDependencyGraph(graph, package, affectedPackages) { + // Terminate early if the package has been visited. if (affectedPackages.has(package)) { return; } diff --git a/tfjs-core/src/backends/non_max_suppression_impl.ts b/tfjs-core/src/backends/non_max_suppression_impl.ts index 351f53e4451..3f7bb8d8961 100644 --- a/tfjs-core/src/backends/non_max_suppression_impl.ts +++ b/tfjs-core/src/backends/non_max_suppression_impl.ts @@ -48,7 +48,7 @@ export function nonMaxSuppressionV5( iouThreshold: number, scoreThreshold: number, softNmsSigma: number): NamedTensorMap { // For NonMaxSuppressionV5Op, we always return a second output holding - // corresponding scores + // corresponding scores. const returnScoresTensor = true; const result = nonMaxSuppressionImpl_( From b52cb9dbb5a5ccca7dfac83c0d6d3e50d65fcade Mon Sep 17 00:00:00 2001 From: Na Li Date: Wed, 26 Feb 2020 17:21:40 -0800 Subject: [PATCH 15/26] Change let to const. --- scripts/diff.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/diff.js b/scripts/diff.js index a8c1b410ba6..51fb235eadf 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -107,7 +107,7 @@ console.log(); // Break up the console for readability. // Only add affected packages if not triggering all builds. if (!triggerAllBuilds) { - let dependencyGraph = constructDependencyGraph('scripts/dependency.json'); + const dependencyGraph = constructDependencyGraph('scripts/dependency.json'); triggeredBuilds.forEach(triggeredBuild => { const affectedPackages = calculateAffectedPackages(dependencyGraph, triggeredBuild); From fc3abe1baacea9252522048b4b84dd1c75b4c953 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 09:59:00 -0800 Subject: [PATCH 16/26] Simplify dependency.json and debug. --- scripts/dependency.json | 12 ---------- scripts/diff.js | 22 +++++++++++-------- scripts/package_dependencies.json | 5 +++++ scripts/test-util.js | 14 ++++++------ .../src/backends/non_max_suppression_impl.ts | 2 +- 5 files changed, 26 insertions(+), 29 deletions(-) delete mode 100644 scripts/dependency.json create mode 100644 scripts/package_dependencies.json diff --git a/scripts/dependency.json b/scripts/dependency.json deleted file mode 100644 index 464c1f08c93..00000000000 --- a/scripts/dependency.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "packages": [ - { - "package": "tfjs-core", - "dependencies": [] - }, - { - "package": "tfjs-converter", - "dependencies": ["tfjs-core"] - } - ] -} diff --git a/scripts/diff.js b/scripts/diff.js index 51fb235eadf..e76b1fd2759 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -14,8 +14,11 @@ // limitations under the License. // ============================================================================= -const {exec, constructDependencyGraph, calculateAffectedPackages} = - require('./test-util'); +const { + exec, + constructDependencyGraph, + computeAffectedPackages: computeAffectedPackages +} = require('./test-util'); const shell = require('shelljs'); const {readdirSync, statSync, writeFileSync} = require('fs'); const {join} = require('path'); @@ -23,7 +26,7 @@ const fs = require('fs'); const filesWhitelistToTriggerBuild = [ 'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json', - 'scripts/diff.js', 'scripts/run-build.sh' + 'scripts/run-build.sh' ]; const CLONE_PATH = 'clone'; @@ -107,18 +110,19 @@ console.log(); // Break up the console for readability. // Only add affected packages if not triggering all builds. if (!triggerAllBuilds) { - const dependencyGraph = constructDependencyGraph('scripts/dependency.json'); + const affectedBuilds = new Set(); + const dependencyGraph = + constructDependencyGraph('scripts/package_dependencies.json'); triggeredBuilds.forEach(triggeredBuild => { const affectedPackages = - calculateAffectedPackages(dependencyGraph, triggeredBuild); + computeAffectedPackages(dependencyGraph, triggeredBuild); affectedPackages.forEach(package => { - writeFileSync(join(package, 'diff'), ''); - triggeredBuilds.push(package); + writeFileSync(join(package, 'diff')); + affectedBuilds.add(package); }); }); - // Deduplicate the triggered builds for log. - triggeredBuilds = [...new Set(triggeredBuilds)]; + triggeredBuilds.push(Array.from(affectedBuilds)); } // Filter the triggered builds to log by whether a cloudbuild.yml file diff --git a/scripts/package_dependencies.json b/scripts/package_dependencies.json new file mode 100644 index 00000000000..104b0980b37 --- /dev/null +++ b/scripts/package_dependencies.json @@ -0,0 +1,5 @@ + +{ + "tfjs-core": [], + "tfjs-converter": ["tfjs-core"] +} diff --git a/scripts/test-util.js b/scripts/test-util.js index 89e18650939..ed8cc02f9c5 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -38,22 +38,22 @@ function constructDependencyGraph(dependencyFilePath) { const dependencyGraph = {}; - dependencyInfo.packages.forEach( - package => {package.dependencies.forEach(dependency => { + Object.keys(dependencyInfo) + .forEach(package => dependencyInfo[package].forEach(dependency => { if (!dependencyGraph[dependency]) { dependencyGraph[dependency] = []; } - dependencyGraph[dependency].push(package.package); - })}); + dependencyGraph[dependency].push(package); + })); return dependencyGraph; } -function calculateAffectedPackages(dependencyGraph, package) { +function computeAffectedPackages(dependencyGraph, package) { const affectedPackages = new Set(); traverseDependencyGraph(dependencyGraph, package, affectedPackages); - return [...affectedPackages]; + return Array.from(affectedPackages); } // This function performs a depth-first-search to add affected packages that @@ -78,4 +78,4 @@ function traverseDependencyGraph(graph, package, affectedPackages) { exports.exec = exec; exports.constructDependencyGraph = constructDependencyGraph; -exports.calculateAffectedPackages = calculateAffectedPackages; +exports.calculateAffectedPackages = computeAffectedPackages; diff --git a/tfjs-core/src/backends/non_max_suppression_impl.ts b/tfjs-core/src/backends/non_max_suppression_impl.ts index 3f7bb8d8961..351f53e4451 100644 --- a/tfjs-core/src/backends/non_max_suppression_impl.ts +++ b/tfjs-core/src/backends/non_max_suppression_impl.ts @@ -48,7 +48,7 @@ export function nonMaxSuppressionV5( iouThreshold: number, scoreThreshold: number, softNmsSigma: number): NamedTensorMap { // For NonMaxSuppressionV5Op, we always return a second output holding - // corresponding scores. + // corresponding scores const returnScoresTensor = true; const result = nonMaxSuppressionImpl_( From 7f85117122b8515f4cbeca052f5cface01704f74 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 10:12:49 -0800 Subject: [PATCH 17/26] . --- scripts/diff.js | 7 ++----- scripts/test-util.js | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/diff.js b/scripts/diff.js index e76b1fd2759..769aca10fde 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -14,11 +14,8 @@ // limitations under the License. // ============================================================================= -const { - exec, - constructDependencyGraph, - computeAffectedPackages: computeAffectedPackages -} = require('./test-util'); +const {exec, constructDependencyGraph, computeAffectedPackages} = + require('./test-util'); const shell = require('shelljs'); const {readdirSync, statSync, writeFileSync} = require('fs'); const {join} = require('path'); diff --git a/scripts/test-util.js b/scripts/test-util.js index ed8cc02f9c5..d2d3b085a0e 100644 --- a/scripts/test-util.js +++ b/scripts/test-util.js @@ -78,4 +78,4 @@ function traverseDependencyGraph(graph, package, affectedPackages) { exports.exec = exec; exports.constructDependencyGraph = constructDependencyGraph; -exports.calculateAffectedPackages = computeAffectedPackages; +exports.computeAffectedPackages = computeAffectedPackages; From 132104fbf578ca12738374dc07c6ad8b85a9a08e Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 10:28:14 -0800 Subject: [PATCH 18/26] Final clean up. --- scripts/diff.js | 2 +- tfjs-core/src/backends/non_max_suppression_impl.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/diff.js b/scripts/diff.js index 769aca10fde..ec06a4790ca 100755 --- a/scripts/diff.js +++ b/scripts/diff.js @@ -23,7 +23,7 @@ const fs = require('fs'); const filesWhitelistToTriggerBuild = [ 'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json', - 'scripts/run-build.sh' + 'scripts/diff.js', 'scripts/run-build.sh' ]; const CLONE_PATH = 'clone'; diff --git a/tfjs-core/src/backends/non_max_suppression_impl.ts b/tfjs-core/src/backends/non_max_suppression_impl.ts index 351f53e4451..3f7bb8d8961 100644 --- a/tfjs-core/src/backends/non_max_suppression_impl.ts +++ b/tfjs-core/src/backends/non_max_suppression_impl.ts @@ -48,7 +48,7 @@ export function nonMaxSuppressionV5( iouThreshold: number, scoreThreshold: number, softNmsSigma: number): NamedTensorMap { // For NonMaxSuppressionV5Op, we always return a second output holding - // corresponding scores + // corresponding scores. const returnScoresTensor = true; const result = nonMaxSuppressionImpl_( From 82401bbc8e3ee3658ae4f32d5c976998dbbd7f98 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 14:12:55 -0800 Subject: [PATCH 19/26] Change diff.js to find-affected-packages.js and change diff file to run-ci. Also debug. --- package.json | 2 +- scripts/{diff.js => find-affected-packages.js} | 8 ++++---- tfjs-core/src/backends/non_max_suppression_impl.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename scripts/{diff.js => find-affected-packages.js} (96%) diff --git a/package.json b/package.json index 4028bd997b9..b33988fea3b 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "typescript": "3.5.3" }, "scripts": { - "diff": "./scripts/diff.js", + "diff": "./scripts/find-affected-packages.js", "release": "ts-node ./scripts/release.ts", "release-notes": "ts-node ./scripts/release_notes/release_notes.ts", "test-release-notes": "ts-node ./scripts/release_notes/run_tests.ts" diff --git a/scripts/diff.js b/scripts/find-affected-packages.js similarity index 96% rename from scripts/diff.js rename to scripts/find-affected-packages.js index ec06a4790ca..a5e0744f00d 100755 --- a/scripts/diff.js +++ b/scripts/find-affected-packages.js @@ -23,7 +23,7 @@ const fs = require('fs'); const filesWhitelistToTriggerBuild = [ 'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json', - 'scripts/diff.js', 'scripts/run-build.sh' + 'scripts/run-build.sh' ]; const CLONE_PATH = 'clone'; @@ -87,7 +87,7 @@ console.log(); // Break up the console for readability. let triggeredBuilds = []; dirs.forEach(dir => { - shell.rm('-f', `${dir}/diff`); + shell.rm('-f', `${dir}/run-ci`); const diffOutput = diff(`${dir}/`); if (diffOutput !== '') { console.log(`${dir} has modified files.`); @@ -98,7 +98,7 @@ dirs.forEach(dir => { const shouldDiff = diffOutput !== '' || triggerAllBuilds; if (shouldDiff) { const diffContents = whitelistDiffOutput.join('\n') + '\n' + diffOutput; - writeFileSync(join(dir, 'diff'), diffContents); + writeFileSync(join(dir, 'run-ci'), diffContents); triggeredBuilds.push(dir); } }); @@ -114,7 +114,7 @@ if (!triggerAllBuilds) { const affectedPackages = computeAffectedPackages(dependencyGraph, triggeredBuild); affectedPackages.forEach(package => { - writeFileSync(join(package, 'diff')); + writeFileSync(join(package, 'run-ci')); affectedBuilds.add(package); }); }); diff --git a/tfjs-core/src/backends/non_max_suppression_impl.ts b/tfjs-core/src/backends/non_max_suppression_impl.ts index 3f7bb8d8961..351f53e4451 100644 --- a/tfjs-core/src/backends/non_max_suppression_impl.ts +++ b/tfjs-core/src/backends/non_max_suppression_impl.ts @@ -48,7 +48,7 @@ export function nonMaxSuppressionV5( iouThreshold: number, scoreThreshold: number, softNmsSigma: number): NamedTensorMap { // For NonMaxSuppressionV5Op, we always return a second output holding - // corresponding scores. + // corresponding scores const returnScoresTensor = true; const result = nonMaxSuppressionImpl_( From 01ce44548debf605f3fa68512b58eb2f096b3b61 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 14:25:17 -0800 Subject: [PATCH 20/26] Change cloudbuild diff to find-affected-packages. --- .gitignore | 1 + cloudbuild.yml | 30 +++++++++++++++--------------- package.json | 2 +- scripts/find-affected-packages.js | 1 + scripts/run-build.sh | 2 +- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 0417310c11b..6fd7cc7df40 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.tgz *.zip */diff +*/run-ci **/*.pyc **/bundle.js **/*.js.map diff --git a/cloudbuild.yml b/cloudbuild.yml index 0fc1d8098d7..fa3d01075fc 100644 --- a/cloudbuild.yml +++ b/cloudbuild.yml @@ -5,11 +5,11 @@ steps: id: 'yarn' args: ['install'] -# Run diff to find modified files in each folder. +# Run find-affected-packages to find affected files in each folder. - name: 'node:10' entrypoint: 'yarn' - id: 'diff' - args: ['diff'] + id: 'find-affected-packages' + args: ['find-affected-packages'] waitFor: ['yarn'] env: - 'COMMIT_SHA=$COMMIT_SHA' @@ -20,77 +20,77 @@ steps: entrypoint: 'bash' id: 'tfjs-core' args: ['./scripts/run-build.sh', 'tfjs-core'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Converter. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-converter' args: ['./scripts/run-build.sh', 'tfjs-converter'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Data. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-data' args: ['./scripts/run-build.sh', 'tfjs-data'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Layers. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-layers' args: ['./scripts/run-build.sh', 'tfjs-layers'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Union. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs' args: ['./scripts/run-build.sh', 'tfjs'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Vis. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-vis' args: ['./scripts/run-build.sh', 'tfjs-vis'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # WebGPU. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-backend-webgpu' args: ['./scripts/run-build.sh', 'tfjs-backend-webgpu'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # WASM. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-backend-wasm' args: ['./scripts/run-build.sh', 'tfjs-backend-wasm'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # React Native. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-react-native' args: ['./scripts/run-build.sh', 'tfjs-react-native'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Node CPU. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-node' args: ['./scripts/run-build.sh', 'tfjs-node'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Node GPU. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' id: 'tfjs-node-gpu' args: ['./scripts/run-build.sh', 'tfjs-node-gpu'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] # Integration tests - name: 'node:10' @@ -98,7 +98,7 @@ steps: id: 'test-integration' entrypoint: 'yarn' args: ['test-integration'] - waitFor: ['diff'] + waitFor: ['find-affected-packages'] env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY'] secretEnv: ['BROWSERSTACK_KEY'] diff --git a/package.json b/package.json index b33988fea3b..15f5c24eb42 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "typescript": "3.5.3" }, "scripts": { - "diff": "./scripts/find-affected-packages.js", + "find-affected-packages": "./scripts/find-affected-packages.js", "release": "ts-node ./scripts/release.ts", "release-notes": "ts-node ./scripts/release_notes/release_notes.ts", "test-release-notes": "ts-node ./scripts/release_notes/run_tests.ts" diff --git a/scripts/find-affected-packages.js b/scripts/find-affected-packages.js index a5e0744f00d..5b347b27c60 100755 --- a/scripts/find-affected-packages.js +++ b/scripts/find-affected-packages.js @@ -107,6 +107,7 @@ console.log(); // Break up the console for readability. // Only add affected packages if not triggering all builds. if (!triggerAllBuilds) { + console.log('Computing affected packages.'); const affectedBuilds = new Set(); const dependencyGraph = constructDependencyGraph('scripts/package_dependencies.json'); diff --git a/scripts/run-build.sh b/scripts/run-build.sh index 56fe96ee219..c5dabdf7595 100755 --- a/scripts/run-build.sh +++ b/scripts/run-build.sh @@ -17,6 +17,6 @@ set -e DIR=$1 -if test -f "$DIR/diff"; then +if test -f "$DIR/run-ci"; then gcloud builds submit . --config=$DIR/cloudbuild.yml fi From d3a47d9b0b76e84e644bf3c2ada5096df2aac6d7 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 14:29:58 -0800 Subject: [PATCH 21/26] debug. --- scripts/find-affected-packages.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/find-affected-packages.js b/scripts/find-affected-packages.js index 5b347b27c60..8d6b0e301e9 100755 --- a/scripts/find-affected-packages.js +++ b/scripts/find-affected-packages.js @@ -21,10 +21,8 @@ const {readdirSync, statSync, writeFileSync} = require('fs'); const {join} = require('path'); const fs = require('fs'); -const filesWhitelistToTriggerBuild = [ - 'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json', - 'scripts/run-build.sh' -]; +const filesWhitelistToTriggerBuild = + ['package.json', 'tsconfig.json', 'tslint.json']; const CLONE_PATH = 'clone'; From fcafe73a92e85ed2451f38c8f0e6ade7f59cec83 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 14:40:46 -0800 Subject: [PATCH 22/26] debug. --- scripts/find-affected-packages.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/find-affected-packages.js b/scripts/find-affected-packages.js index 8d6b0e301e9..80904c32340 100755 --- a/scripts/find-affected-packages.js +++ b/scripts/find-affected-packages.js @@ -21,8 +21,7 @@ const {readdirSync, statSync, writeFileSync} = require('fs'); const {join} = require('path'); const fs = require('fs'); -const filesWhitelistToTriggerBuild = - ['package.json', 'tsconfig.json', 'tslint.json']; +const filesWhitelistToTriggerBuild = []; const CLONE_PATH = 'clone'; From 5330717aa242ddf8ef7cc11f42622bc10ff6d59e Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 14:45:07 -0800 Subject: [PATCH 23/26] Clean up. --- scripts/find-affected-packages.js | 5 ++++- tfjs-core/src/backends/non_max_suppression_impl.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/find-affected-packages.js b/scripts/find-affected-packages.js index 80904c32340..24b33acdcc5 100755 --- a/scripts/find-affected-packages.js +++ b/scripts/find-affected-packages.js @@ -21,7 +21,10 @@ const {readdirSync, statSync, writeFileSync} = require('fs'); const {join} = require('path'); const fs = require('fs'); -const filesWhitelistToTriggerBuild = []; +const filesWhitelistToTriggerBuild = [ + 'cloudbuild.yml', 'package.json', 'tsconfig.json', 'tslint.json', + 'scripts/find-affected-packages.js', 'scripts/run-build.sh' +]; const CLONE_PATH = 'clone'; diff --git a/tfjs-core/src/backends/non_max_suppression_impl.ts b/tfjs-core/src/backends/non_max_suppression_impl.ts index 351f53e4451..3f7bb8d8961 100644 --- a/tfjs-core/src/backends/non_max_suppression_impl.ts +++ b/tfjs-core/src/backends/non_max_suppression_impl.ts @@ -48,7 +48,7 @@ export function nonMaxSuppressionV5( iouThreshold: number, scoreThreshold: number, softNmsSigma: number): NamedTensorMap { // For NonMaxSuppressionV5Op, we always return a second output holding - // corresponding scores + // corresponding scores. const returnScoresTensor = true; const result = nonMaxSuppressionImpl_( From 0f866422d56fd32826529873a20d2c6e9c5c0ecf Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 15:48:48 -0800 Subject: [PATCH 24/26] Update layers integration test to python3. --- tfjs-layers/scripts/tfjs2keras-py.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tfjs-layers/scripts/tfjs2keras-py.sh b/tfjs-layers/scripts/tfjs2keras-py.sh index a696a4e2146..2756110a4cc 100755 --- a/tfjs-layers/scripts/tfjs2keras-py.sh +++ b/tfjs-layers/scripts/tfjs2keras-py.sh @@ -43,13 +43,13 @@ fi VENV_DIR="$(mktemp -d)_venv" echo "Creating virtualenv at ${VENV_DIR} ..." -virtualenv "${VENV_DIR}" +virtualenv -p python3 "${VENV_DIR}" source "${VENV_DIR}/bin/activate" if [[ "${DEV_VERSION}" == "stable" ]]; then - pip install -r requirements-stable.txt + pip3 install -r requirements-stable.txt else - pip install -r requirements-dev.txt + pip3 install -r requirements-dev.txt fi export TFJS2KERAS_TEST_USING_TF_KERAS="${TFJS2KERAS_TEST_USING_TF_KERAS}" From 5b338152b5faca7b4b95b799cd91712ed1733ac4 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 16:07:59 -0800 Subject: [PATCH 25/26] Remove tf-nightly test in layers because unresolved version issues. --- tfjs-layers/cloudbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index b2ac993e1bd..1a96528c8fe 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -34,7 +34,7 @@ steps: dir: 'tfjs-layers' entrypoint: 'bash' id: 'tfjs2keras-py' - args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras && ./scripts/tfjs2keras-py.sh --dev --tfkeras'] + args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras] waitFor: ['tfjs2keras-js'] - name: 'node:10' dir: 'tfjs-layers' From c9e74f8daa8c1aafced771f5eef65123150f01d6 Mon Sep 17 00:00:00 2001 From: Na Li Date: Fri, 28 Feb 2020 16:10:13 -0800 Subject: [PATCH 26/26] . --- tfjs-layers/cloudbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-layers/cloudbuild.yml b/tfjs-layers/cloudbuild.yml index 1a96528c8fe..07d33046212 100644 --- a/tfjs-layers/cloudbuild.yml +++ b/tfjs-layers/cloudbuild.yml @@ -34,7 +34,7 @@ steps: dir: 'tfjs-layers' entrypoint: 'bash' id: 'tfjs2keras-py' - args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras] + args: ['-c', './scripts/tfjs2keras-py.sh --stable && ./scripts/tfjs2keras-py.sh --stable --tfkeras'] waitFor: ['tfjs2keras-js'] - name: 'node:10' dir: 'tfjs-layers'