Skip to content

[node] Clean up dependency of tfc and tfl in node tests #2995

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 1, 2020
Merged

[node] Clean up dependency of tfc and tfl in node tests #2995

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cb9a2d3
clean up tfc&tfl
Mar 30, 2020
ab1da34
Merge branch 'master' of github.com:tensorflow/tfjs into clean-tfc-tf…
Apr 1, 2020
c4e02ca
remove unnecessary deps on tfjs-core
Apr 1, 2020
d233cdd
save
Apr 1, 2020
87b2e9c
add tfjs-core back
Apr 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/release-util.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ export const UNION_PHASE: Phase = {
// the test to tf.layers.
export const NODE_PHASE: Phase = {
packages: ['tfjs-node', 'tfjs-node-gpu'],
deps: ['tfjs', 'tfjs-core', 'tfjs-layers'],
deps: ['tfjs', 'tfjs-core'],
scripts: {'tfjs-node-gpu': {'before-yarn': ['yarn prep-gpu']}}
};

Expand Down
27 changes: 4 additions & 23 deletions tfjs-core/yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,29 +18,10 @@
esutils "^2.0.2"
js-tokens "^4.0.0"

"@bazel/bazel-darwin_x64@0.24.0":
version "0.24.0"
resolved "https://registry.yarnpkg.com/@bazel/bazel-darwin_x64/-/bazel-darwin_x64-0.24.0.tgz#828ef298d8d542961df388f17b0244f4f4302a74"
integrity sha512-xly44vkcD/fauUb7Lm5Lme4qhEZdkuuyBKSVQUHPbYAGDdbj/W8dupI3bZREkJAgG/WrRU+WXUemMj4U8ZcLcw==

"@bazel/bazel-linux_x64@0.24.0":
version "0.24.0"
resolved "https://registry.yarnpkg.com/@bazel/bazel-linux_x64/-/bazel-linux_x64-0.24.0.tgz#9ef2e7266833ad2221fe4af4ceb6763d2897e3ff"
integrity sha512-p5ylPLWnJZDGbaIFBrtD/tp3Su5rMdzeeNJKU24XyiWQTHVZ3OD3I2Fb0ILCgfBjY8AlA7EtCtOI4hYnAuIOtg==

"@bazel/bazel-win32_x64@0.24.0":
version "0.24.0"
resolved "https://registry.yarnpkg.com/@bazel/bazel-win32_x64/-/bazel-win32_x64-0.24.0.tgz#02d83113a6c6ed99795a3e41bff5631aa141638d"
integrity sha512-/bcSEx+GoV/q7H4WM0jazfxTcurSiIIePhRv+d05mxRDcaWwhCO8KzmmZRWH1abW6npvq5tLkbSQi7G7nUBhgg==

"@bazel/bazel@^0.24.0":
version "0.24.0"
resolved "https://registry.yarnpkg.com/@bazel/bazel/-/bazel-0.24.0.tgz#f4e68e3680ac299858c24c26be3d08d1151e78fc"
integrity sha512-/5E55tqH9ogAGF9Dd7RSCJmk7/xdlsPTAhsX3yEsEMs7GLdHlgD3jbeePsKUiHKKr8LXAufjTs2pXQfjrkZRMg==
optionalDependencies:
"@bazel/bazel-darwin_x64" "0.24.0"
"@bazel/bazel-linux_x64" "0.24.0"
"@bazel/bazel-win32_x64" "0.24.0"
"@bazel/bazelisk@^1.3.0":
Copy link
Collaborator

@lina128 lina128 Apr 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi Kangyi, is this change intended?

Copy link
Contributor Author

@kangyizhang kangyizhang Apr 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this is an update based on #2912

Copy link
Collaborator

@lina128 lina128 Apr 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it. Thanks!

version "1.3.0"
resolved "https://registry.yarnpkg.com/@bazel/bazelisk/-/bazelisk-1.3.0.tgz#dc312dd30ad01e9af86e53b40795ab6e545fa55b"
integrity sha512-73H1nq3572tTf+dhDT86aWQN+LCyfxrh05jabqPXp6cpR8soxte3gS5oUqkN36fUe+J2HzNiV4CXZTz4Xytd3Q==

"@bazel/typescript@^0.27.8":
version "0.27.10"
Expand Down
1 change: 0 additions & 1 deletion tfjs-node/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
"upload-windows-addon": "./scripts/build-and-upload-windows-addon.bat"
},
"devDependencies": {
"@tensorflow/tfjs-layers": "link:../tfjs-layers",
"@types/jasmine": "~2.8.6",
"@types/node": "^10.5.1",
"@types/progress": "^2.0.1",
Expand Down
2 changes: 1 addition & 1 deletion tfjs-node/src/canvas_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* =============================================================================
*/

import * as tf from '@tensorflow/tfjs-core';
import * as tf from '@tensorflow/tfjs';

class MockContext {
getImageData(x: number, y: number, width: number, height: number) {
Expand Down
3 changes: 1 addition & 2 deletions tfjs-node/src/image.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@
* =============================================================================
*/

import {Tensor} from '@tensorflow/tfjs';
import {Tensor3D, Tensor4D, tidy, util} from '@tensorflow/tfjs-core';
import {Tensor, Tensor3D, Tensor4D, tidy, util} from '@tensorflow/tfjs';
import {ensureTensorflowBackend, nodeBackend} from './nodejs_kernel_backend';

export enum ImageType {
Expand Down
2 changes: 1 addition & 1 deletion tfjs-node/src/image_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* =============================================================================
*/

import {memory, setBackend, test_util} from '@tensorflow/tfjs-core';
import {memory, setBackend, test_util} from '@tensorflow/tfjs';
import * as fs from 'fs';
import {promisify} from 'util';
import {getImageType, ImageType} from './image';
Expand Down
30 changes: 15 additions & 15 deletions tfjs-node/src/io/file_system.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* =============================================================================
*/

import * as tfc from '@tensorflow/tfjs-core';
import * as tf from '@tensorflow/tfjs';
import * as fs from 'fs';
import {dirname, join, resolve} from 'path';
import {promisify} from 'util';
Expand All @@ -39,7 +39,7 @@ function doesNotExistHandler(name: string): (e: NodeJS.ErrnoException) =>
};
}

export class NodeFileSystem implements tfc.io.IOHandler {
export class NodeFileSystem implements tf.io.IOHandler {
static readonly URL_SCHEME = 'file://';

protected readonly path: string|string[];
Expand Down Expand Up @@ -68,7 +68,7 @@ export class NodeFileSystem implements tfc.io.IOHandler {
*/
constructor(path: string|string[]) {
if (Array.isArray(path)) {
tfc.util.assert(
tf.util.assert(
path.length === 2,
() => 'file paths must have a length of 2, ' +
`(actual length is ${path.length}).`);
Expand All @@ -78,8 +78,8 @@ export class NodeFileSystem implements tfc.io.IOHandler {
}
}

async save(modelArtifacts: tfc.io.ModelArtifacts):
Promise<tfc.io.SaveResult> {
async save(modelArtifacts: tf.io.ModelArtifacts):
Promise<tf.io.SaveResult> {
if (Array.isArray(this.path)) {
throw new Error('Cannot perform saving to multiple paths.');
}
Expand All @@ -98,7 +98,7 @@ export class NodeFileSystem implements tfc.io.IOHandler {
paths: [this.WEIGHTS_BINARY_FILENAME],
weights: modelArtifacts.weightSpecs
}];
const modelJSON: tfc.io.ModelJSON = {
const modelJSON: tf.io.ModelJSON = {
modelTopology: modelArtifacts.modelTopology,
weightsManifest,
format: modelArtifacts.format,
Expand All @@ -117,19 +117,19 @@ export class NodeFileSystem implements tfc.io.IOHandler {
weightsBinPath, Buffer.from(modelArtifacts.weightData), 'binary');

return {
// TODO(cais): Use explicit tfc.io.ModelArtifactsInfo type below once it
// TODO(cais): Use explicit tf.io.ModelArtifactsInfo type below once it
// is available.
// tslint:disable-next-line:no-any
modelArtifactsInfo: getModelArtifactsInfoForJSON(modelArtifacts) as any
};
}
}
async load(): Promise<tfc.io.ModelArtifacts> {
async load(): Promise<tf.io.ModelArtifacts> {
return Array.isArray(this.path) ? this.loadBinaryModel() :
this.loadJSONModel();
}

protected async loadBinaryModel(): Promise<tfc.io.ModelArtifacts> {
protected async loadBinaryModel(): Promise<tf.io.ModelArtifacts> {
const topologyPath = this.path[0];
const weightManifestPath = this.path[1];
const topology =
Expand All @@ -150,7 +150,7 @@ export class NodeFileSystem implements tfc.io.IOHandler {
const modelTopology = await readFile(this.path[0]);
const weightsManifest = JSON.parse(await readFile(this.path[1], 'utf8'));

const modelArtifacts: tfc.io.ModelArtifacts = {
const modelArtifacts: tf.io.ModelArtifacts = {
modelTopology,
};
const [weightSpecs, weightData] =
Expand All @@ -162,7 +162,7 @@ export class NodeFileSystem implements tfc.io.IOHandler {
return modelArtifacts;
}

protected async loadJSONModel(): Promise<tfc.io.ModelArtifacts> {
protected async loadJSONModel(): Promise<tf.io.ModelArtifacts> {
const path = this.path as string;
const info = await stat(path).catch(doesNotExistHandler('Path'));

Expand All @@ -171,7 +171,7 @@ export class NodeFileSystem implements tfc.io.IOHandler {
if (info.isFile()) {
const modelJSON = JSON.parse(await readFile(path, 'utf8'));

const modelArtifacts: tfc.io.ModelArtifacts = {
const modelArtifacts: tf.io.ModelArtifacts = {
modelTopology: modelJSON.modelTopology,
format: modelJSON.format,
generatedBy: modelJSON.generatedBy,
Expand All @@ -198,11 +198,11 @@ export class NodeFileSystem implements tfc.io.IOHandler {
}

private async loadWeights(
weightsManifest: tfc.io.WeightsManifestConfig,
path: string): Promise<[tfc.io.WeightsManifestEntry[], ArrayBuffer]> {
weightsManifest: tf.io.WeightsManifestConfig,
path: string): Promise<[tf.io.WeightsManifestEntry[], ArrayBuffer]> {
const dirName = dirname(path);
const buffers: Buffer[] = [];
const weightSpecs: tfc.io.WeightsManifestEntry[] = [];
const weightSpecs: tf.io.WeightsManifestEntry[] = [];
for (const group of weightsManifest) {
for (const path of group.paths) {
const weightFilePath = join(dirName, path);
Expand Down
52 changes: 25 additions & 27 deletions tfjs-node/src/io/file_system_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
* =============================================================================
*/

import * as tfc from '@tensorflow/tfjs-core';
import {test_util} from '@tensorflow/tfjs-core';
import * as tfl from '@tensorflow/tfjs-layers';
import * as tf from '@tensorflow/tfjs';
import * as fs from 'fs';
import * as path from 'path';
import * as rimraf from 'rimraf';
Expand Down Expand Up @@ -65,7 +63,7 @@ describe('File system IOHandler', () => {
}],
'backend': 'tensorflow'
};
const weightSpecs1: tfc.io.WeightsManifestEntry[] = [
const weightSpecs1: tf.io.WeightsManifestEntry[] = [
{
name: 'dense/kernel',
shape: [3, 1],
Expand Down Expand Up @@ -93,7 +91,7 @@ describe('File system IOHandler', () => {
it('save succeeds with newly created directory', async done => {
const t0 = new Date();
const dir = path.join(testDir, 'save-destination');
const handler = tfc.io.getSaveHandlers(`file://${dir}`)[0];
const handler = tf.io.getSaveHandlers(`file://${dir}`)[0];
handler
.save({
modelTopology: modelTopology1,
Expand Down Expand Up @@ -128,7 +126,7 @@ describe('File system IOHandler', () => {
const dir = path.join(testDir, 'save-destination');
// Create a file at the locatin.
await writeFile(dir, 'foo');
const handler = tfc.io.getSaveHandlers(`file://${dir}`)[0];
const handler = tf.io.getSaveHandlers(`file://${dir}`)[0];
handler
.save({
modelTopology: modelTopology1,
Expand All @@ -145,7 +143,7 @@ describe('File system IOHandler', () => {
});

it('save-load round trip: one weight file', done => {
const handler1 = tfc.io.getSaveHandlers(`file://${testDir}`)[0];
const handler1 = tf.io.getSaveHandlers(`file://${testDir}`)[0];
handler1
.save({
modelTopology: modelTopology1,
Expand All @@ -154,7 +152,7 @@ describe('File system IOHandler', () => {
})
.then(saveResult => {
const modelJSONPath = path.join(testDir, 'model.json');
const handler2 = tfc.io.getLoadHandlers(`file://${modelJSONPath}`)[0];
const handler2 = tf.io.getLoadHandlers(`file://${modelJSONPath}`)[0];
handler2.load()
.then(modelArtifacts => {
expect(modelArtifacts.modelTopology).toEqual(modelTopology1);
Expand All @@ -170,7 +168,7 @@ describe('File system IOHandler', () => {

describe('load json model', () => {
it('load: two weight files', async done => {
const weightsManifest: tfc.io.WeightsManifestConfig = [
const weightsManifest: tf.io.WeightsManifestConfig = [
{
paths: ['weights.1.bin'],
weights: [{
Expand Down Expand Up @@ -209,7 +207,7 @@ describe('File system IOHandler', () => {

// Load the artifacts consisting of a model.json and two binary weight
// files.
const handler = tfc.io.getLoadHandlers(`file://${modelJSONPath}`)[0];
const handler = tf.io.getLoadHandlers(`file://${modelJSONPath}`)[0];
handler.load()
.then(modelArtifacts => {
expect(modelArtifacts.modelTopology).toEqual(modelTopology1);
Expand All @@ -225,7 +223,7 @@ describe('File system IOHandler', () => {
dtype: 'float32',
}
]);
test_util.expectArraysClose(
tf.test_util.expectArraysClose(
new Float32Array(modelArtifacts.weightData),
new Float32Array([-1.1, -3.3, -3.3, -7.7]));
done();
Expand All @@ -235,7 +233,7 @@ describe('File system IOHandler', () => {

it('loading from nonexistent model.json path fails', done => {
const handler =
tfc.io.getLoadHandlers(`file://${testDir}/foo/model.json`)[0];
tf.io.getLoadHandlers(`file://${testDir}/foo/model.json`)[0];
handler.load()
.then(getModelArtifactsInfoForJSON => {
done.fail(
Expand All @@ -250,7 +248,7 @@ describe('File system IOHandler', () => {
});

it('loading from missing weights path fails', async done => {
const weightsManifest: tfc.io.WeightsManifestConfig = [
const weightsManifest: tf.io.WeightsManifestConfig = [
{
paths: ['weights.1.bin'],
weights: [{
Expand Down Expand Up @@ -286,7 +284,7 @@ describe('File system IOHandler', () => {

// Load the artifacts consisting of a model.json and two binary weight
// files.
const handler = tfc.io.getLoadHandlers(`file://${modelJSONPath}`)[0];
const handler = tf.io.getLoadHandlers(`file://${modelJSONPath}`)[0];
handler.load()
.then(modelArtifacts => {
done.fail(
Expand All @@ -303,7 +301,7 @@ describe('File system IOHandler', () => {

describe('load binary model', () => {
it('load: two weight files', async done => {
const weightsManifest: tfc.io.WeightsManifestConfig = [
const weightsManifest: tf.io.WeightsManifestConfig = [
{
paths: ['weights.1.bin'],
weights: [{
Expand Down Expand Up @@ -348,7 +346,7 @@ describe('File system IOHandler', () => {
new NodeFileSystem([`${modelPath}`, `${modelManifestJSONPath}`]);
handler.load()
.then(modelArtifacts => {
test_util.expectArraysClose(
tf.test_util.expectArraysClose(
new Uint8Array(modelArtifacts.modelTopology as ArrayBuffer),
new Uint8Array(modelData));
expect(modelArtifacts.weightSpecs).toEqual([
Expand All @@ -363,7 +361,7 @@ describe('File system IOHandler', () => {
dtype: 'float32',
}
]);
test_util.expectArraysClose(
tf.test_util.expectArraysClose(
new Float32Array(modelArtifacts.weightData),
new Float32Array([-1.1, -3.3, -3.3, -7.7]));
done();
Expand Down Expand Up @@ -394,7 +392,7 @@ describe('File system IOHandler', () => {
});

it('loading from missing weights path fails', async done => {
const weightsManifest: tfc.io.WeightsManifestConfig = [
const weightsManifest: tf.io.WeightsManifestConfig = [
{
paths: ['weights.1.bin'],
weights: [{
Expand Down Expand Up @@ -454,21 +452,21 @@ describe('File system IOHandler', () => {
});

it('Save and load model with loss and optimizer', async () => {
const model = tfl.sequential();
model.add(tfl.layers.dense(
const model = tf.sequential();
model.add(tf.layers.dense(
{units: 1, kernelInitializer: 'zeros', inputShape: [1]}));
model.compile(
{loss: 'meanSquaredError', optimizer: tfc.train.adam(2.5e-2)});
{loss: 'meanSquaredError', optimizer: tf.train.adam(2.5e-2)});

const xs = tfc.tensor2d([1, 2, 3, 4], [4, 1]);
const ys = tfc.tensor2d([-1, -3, -5, -7], [4, 1]);
const xs = tf.tensor2d([1, 2, 3, 4], [4, 1]);
const ys = tf.tensor2d([-1, -3, -5, -7], [4, 1]);
await model.fit(xs, ys, {epochs: 2, shuffle: false, verbose: 0});

const saveURL = `file://${testDir}`;
const loadURL = `file://${testDir}/model.json`;

await model.save(saveURL, {includeOptimizer: true});
const model2 = await tfl.loadLayersModel(loadURL);
const model2 = await tf.loadLayersModel(loadURL);
const optimizerConfig = model2.optimizer.getConfig();
expect(model2.optimizer.getClassName()).toEqual('Adam');
expect(optimizerConfig['learningRate']).toEqual(2.5e-2);
Expand All @@ -484,16 +482,16 @@ describe('File system IOHandler', () => {
});

it('Save and load model with user-defined metadata', async () => {
const model = tfl.sequential();
model.add(tfl.layers.dense({units: 3, inputShape: [4]}));
const model = tf.sequential();
model.add(tf.layers.dense({units: 3, inputShape: [4]}));
model.setUserDefinedMetadata(
{'outputLabels': ['Label1', 'Label2', 'Label3']});

const saveURL = `file://${testDir}`;
const loadURL = `file://${testDir}/model.json`;

await model.save(saveURL);
const model2 = await tfl.loadLayersModel(loadURL);
const model2 = await tf.loadLayersModel(loadURL);
expect(model2.getUserDefinedMetadata()).toEqual({
'outputLabels': ['Label1', 'Label2', 'Label3']
});
Expand Down
Loading