From 00c16a3ba2088eb670c3b9ca1682b4852f7f265b Mon Sep 17 00:00:00 2001 From: AdamLang96 <45542095+AdamLang96@users.noreply.github.com> Date: Sat, 1 Oct 2022 10:14:07 -0700 Subject: [PATCH 1/5] revision for catencoding pr --- tfjs-layers/package.json | 3 + .../layers/preprocessing/category_encoding.ts | 16 ++-- .../preprocessing/category_encoding_test.ts | 38 ++++++---- .../preprocessing/preprocessing_utils.ts | 50 ++++++++----- .../preprocessing/preprocessing_utils_test.ts | 26 +++++-- tfjs-layers/yarn.lock | 74 +++++++++++++++++++ 6 files changed, 159 insertions(+), 48 deletions(-) diff --git a/tfjs-layers/package.json b/tfjs-layers/package.json index 73c0c0d1c9..31930a4b83 100644 --- a/tfjs-layers/package.json +++ b/tfjs-layers/package.json @@ -35,5 +35,8 @@ }, "peerDependencies": { "@tensorflow/tfjs-core": "link:../link-package/node_modules/@tensorflow/tfjs-core" + }, + "dependencies": { + "@tensorflow/tfjs-core": "^3.20.0" } } diff --git a/tfjs-layers/src/layers/preprocessing/category_encoding.ts b/tfjs-layers/src/layers/preprocessing/category_encoding.ts index 084940a0ea..c6ca0f0040 100644 --- a/tfjs-layers/src/layers/preprocessing/category_encoding.ts +++ b/tfjs-layers/src/layers/preprocessing/category_encoding.ts @@ -17,6 +17,8 @@ import { Kwargs } from '../../types'; import { ValueError } from '../../errors'; import * as K from '../../backend/tfjs_backend'; import * as utils from './preprocessing_utils'; +import { OutputMode } from './preprocessing_utils'; + export declare interface CategoryEncodingArgs extends LayerArgs { numTokens: number; @@ -58,7 +60,7 @@ export class CategoryEncoding extends Layer { return [this.numTokens]; } - if(this.outputMode === utils.oneHot && inputShape[-1] !== 1) { + if(this.outputMode === 'oneHot' && inputShape[inputShape.length - 1] !== 1){ inputShape.push(this.numTokens); return inputShape; } @@ -79,7 +81,7 @@ export class CategoryEncoding extends Layer { if((typeof kwargs['countWeights']) !== 'undefined') { - if(this.outputMode !== utils.count) { + if(this.outputMode !== 'count') { throw new ValueError( `countWeights is not used when outputMode !== count. Received countWeights=${kwargs['countWeights']}`); @@ -93,20 +95,22 @@ export class CategoryEncoding extends Layer { } } - const depth = this.numTokens; const maxValue = max(inputs); const minValue = min(inputs); + const greaterEqualMax = greater(this.numTokens, maxValue) + .bufferSync().get(0); - const greaterEqualMax = greater(depth, maxValue).bufferSync().get(0); const greaterMin = greaterEqual(minValue, 0).bufferSync().get(0); if(!(greaterEqualMax && greaterMin)) { + throw new ValueError( - `Input values must be between 0 < values <= numTokens with numTokens=${this.numTokens}`); + `Input values must be between 0 < values <= numTokens + with numTokens=${this.numTokens}`); } return utils.encodeCategoricalInputs(inputs, - this.outputMode, depth, countWeights); + this.outputMode, this.numTokens, countWeights); }); } } diff --git a/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts b/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts index 8f45f65d7b..c03d7076b8 100644 --- a/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts +++ b/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts @@ -1,7 +1,17 @@ +/** + * @license + * Copyright 2022 CodeSmith LLC + * + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. + * ============================================================================= + */ + import { describeMathCPUAndGPU, expectTensorsClose} from '../../utils/test_utils'; import { Tensor, tensor} from '@tensorflow/tfjs-core'; import { CategoryEncoding } from './category_encoding'; -import * as utils from './preprocessing_utils'; + describeMathCPUAndGPU('Category Encoding', () => { @@ -10,7 +20,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const expectedOutput = tensor([1,0,0,0]); const numTokens = 4; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.count}); + outputMode: 'count'}); const computedOutput = encodingLayer. apply(categoryData) as Tensor; @@ -23,7 +33,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const numTokens = 6; const expectedOutput = tensor([7, 1, 2, 4, 0, 0]); const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.count}); + outputMode: 'count'}); const computedOutput = encodingLayer.apply(categoryData, {countWeights: weightData}) as Tensor; @@ -36,7 +46,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const expectedOutput = tensor([[0, 2, 1, 1, 0, 0], [2, 1, 0, 1, 0, 0]]); const numTokens = 6; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.count}); + outputMode: 'count'}); const computedOutput = encodingLayer.apply(categoryData) as Tensor; expectTensorsClose(computedOutput, expectedOutput); }); @@ -46,7 +56,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const expectedOutput = tensor([0, 0, 0, 1]); const numTokens = 4; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.oneHot}); + outputMode: 'oneHot'}); const computedOutput = encodingLayer.apply(categoryData) as Tensor; expectTensorsClose(computedOutput, expectedOutput); }); @@ -59,7 +69,7 @@ describeMathCPUAndGPU('Category Encoding', () => { [0, 1, 0, 0]]); const numTokens = 4; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.oneHot}); + outputMode: 'oneHot'}); const computedOutput = encodingLayer.apply(categoryData) as Tensor; expectTensorsClose(computedOutput, expectedOutput); }); @@ -72,7 +82,7 @@ describeMathCPUAndGPU('Category Encoding', () => { [0, 1, 0, 0]]); const numTokens = 4; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.oneHot}); + outputMode: 'oneHot'}); const computedOutput = encodingLayer.apply(categoryData) as Tensor; expectTensorsClose(computedOutput, expectedOutput); }); @@ -82,7 +92,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const expectedOutput = tensor([0, 0, 0, 1, 0, 0]); const numTokens = 6; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.multiHot}); + outputMode: 'multiHot'}); const computedOutput = encodingLayer.apply(categoryData) as Tensor; expectTensorsClose(computedOutput, expectedOutput); }); @@ -92,7 +102,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const expectedOutput = tensor([1, 1, 1, 1, 0, 0]); const numTokens = 6; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.multiHot}); + outputMode: 'multiHot'}); const computedOutput = encodingLayer.apply(categoryData) as Tensor; expectTensorsClose(computedOutput, expectedOutput); }); @@ -105,7 +115,7 @@ describeMathCPUAndGPU('Category Encoding', () => { [0, 1, 0, 1]]); const numTokens = 4; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.multiHot}); + outputMode: 'multiHot'}); const computedOutput = encodingLayer.apply(categoryData) as Tensor; expectTensorsClose(computedOutput, expectedOutput); }); @@ -114,10 +124,10 @@ describeMathCPUAndGPU('Category Encoding', () => { const categoryData = tensor([[[1], [2]], [[3], [4]]]); const numTokens = 6; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.multiHot}); + outputMode: 'multiHot'}); expect(() => encodingLayer.apply(categoryData)) .toThrowError(`When outputMode is not 'int', maximum output rank is 2 - Received outputMode ${utils.multiHot} and input shape ${categoryData.shape} + Received outputMode ${'multiHot'} and input shape ${categoryData.shape} which would result in output rank ${categoryData.rank}.`); }); @@ -125,7 +135,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const categoryData = tensor([7, 2, 0, 1]); const numTokens = 3; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.multiHot}); + outputMode: 'multiHot'}); expect(() => encodingLayer.apply(categoryData)) .toThrowError(`Input values must be between 0 < values <= numTokens`); }); @@ -134,7 +144,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const categoryData = tensor([7, 2, -1, 1]); const numTokens = 3; const encodingLayer = new CategoryEncoding({numTokens, - outputMode: utils.multiHot}); + outputMode: 'multiHot'}); expect(() => encodingLayer.apply(categoryData)) .toThrowError(`Input values must be between 0 < values <= numTokens`); }); diff --git a/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts b/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts index 17540d10a5..764fe086d3 100644 --- a/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts +++ b/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts @@ -1,3 +1,13 @@ +/** + * @license + * Copyright 2022 CodeSmith LLC + * + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. + * ============================================================================= + */ + import { Tensor, denseBincount, Tensor1D, Tensor2D, TensorLike, mul} from '@tensorflow/tfjs-core'; import { getExactlyOneTensor } from '../../utils/types_utils'; import { expandDims} from '@tensorflow/tfjs-core'; @@ -7,60 +17,60 @@ import * as K from '../../backend/tfjs_backend'; export type OutputMode = 'int' | 'oneHot' | 'multiHot' | 'count' | 'tfIdf'; export function encodeCategoricalInputs(inputs: Tensor|Tensor[], - outputMode: string, + outputMode: OutputMode, depth: number, weights?: Tensor1D|Tensor2D|TensorLike): Tensor|Tensor[] { let input = getExactlyOneTensor(inputs); - if(inputs.dtype !== 'int32') { - inputs = K.cast(inputs, 'int32'); + if(input.dtype !== 'int32') { + input = K.cast(input, 'int32'); } - if(outputMode === int) { - return inputs; + if(outputMode === 'int') { + return input; } - const originalShape = inputs.shape; + const originalShape = input.shape; - if(inputs.rank === 0) { - inputs = expandDims(inputs, -1); + if(input.rank === 0) { + input = expandDims(input, -1); } - if(outputMode === oneHot) { - if(inputs.shape[inputs.shape.length - 1] !== 1) { - inputs = expandDims(inputs, -1); + if(outputMode === 'oneHot') { + if(input.shape[input.shape.length - 1] !== 1) { + input = expandDims(input, -1); } } - if(inputs.rank > 2) { + if(input.rank > 2) { throw new ValueError(`When outputMode is not 'int', maximum output rank is 2 Received outputMode ${outputMode} and input shape ${originalShape} - which would result in output rank ${inputs.rank}.`); + which would result in output rank ${input.rank}.`); } - const binaryOutput = [multiHot, oneHot].includes(outputMode); + const binaryOutput = ['multiHot', 'oneHot'].includes(outputMode); let denseBincountInput: Tensor1D | Tensor2D; - if(inputs.rank === 1) { - denseBincountInput = inputs as Tensor1D; + if(input.rank === 1) { + denseBincountInput = input as Tensor1D; } - if(inputs.rank === 2) { - denseBincountInput = inputs as Tensor2D; + if(input.rank === 2) { + denseBincountInput = input as Tensor2D; } let binCounts: Tensor1D | Tensor2D; - if ((typeof weights) !== 'undefined' && outputMode === count) { + if ((typeof weights) !== 'undefined' && outputMode === 'count') { binCounts = denseBincount(denseBincountInput, weights, depth, binaryOutput); } else { binCounts = denseBincount(denseBincountInput, [], depth, binaryOutput); } - if(outputMode !== tfIdf) { + if(outputMode !== 'tfIdf') { return binCounts; } diff --git a/tfjs-layers/src/layers/preprocessing/preprocessing_utils_test.ts b/tfjs-layers/src/layers/preprocessing/preprocessing_utils_test.ts index 8d7c479ed3..a6fc5d0c7d 100644 --- a/tfjs-layers/src/layers/preprocessing/preprocessing_utils_test.ts +++ b/tfjs-layers/src/layers/preprocessing/preprocessing_utils_test.ts @@ -1,3 +1,13 @@ +/** + * @license + * Copyright 2022 CodeSmith LLC + * + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. + * ============================================================================= + */ + import { describeMathCPUAndGPU, expectTensorsClose} from '../../utils/test_utils'; import { Tensor, tensor, tensor1d} from '@tensorflow/tfjs-core'; import * as utils from './preprocessing_utils'; @@ -7,14 +17,14 @@ describeMathCPUAndGPU('Tests for preprocessing utils', () => { it('Peforms int encoding correctly', () => { const inputs = tensor([0, 1, 2], [3], 'int32'); const outputs = utils. - encodeCategoricalInputs(inputs, utils.int, 4) as Tensor; + encodeCategoricalInputs(inputs, 'int', 4) as Tensor; expectTensorsClose(outputs, inputs); }); it('Peforms oneHot encoding correctly', () => { const inputs = tensor([0, 1, 2], [3], 'int32'); const outputs = utils. - encodeCategoricalInputs(inputs, utils.oneHot, 4) as Tensor; + encodeCategoricalInputs(inputs, 'oneHot', 4) as Tensor; const expectedOutput = tensor([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]]); expectTensorsClose(outputs, expectedOutput); }); @@ -22,7 +32,7 @@ describeMathCPUAndGPU('Tests for preprocessing utils', () => { it('Peforms multiHot encoding correctly', () => { const inputs = tensor([0, 1, 2], [3], 'int32'); const outputs = utils. - encodeCategoricalInputs(inputs, utils.multiHot, 4) as Tensor; + encodeCategoricalInputs(inputs, 'multiHot', 4) as Tensor; const expectedOutput = tensor([1, 1, 1, 0]); expectTensorsClose(outputs, expectedOutput); }); @@ -30,7 +40,7 @@ describeMathCPUAndGPU('Tests for preprocessing utils', () => { it('Peforms count encoding correctly', () => { const inputs = tensor([0, 1, 1, 2, 2, 2], [6], 'int32'); const outputs = utils. - encodeCategoricalInputs(inputs, utils.count, 4) as Tensor; + encodeCategoricalInputs(inputs, 'count', 4) as Tensor; const expectedOutput = tensor([1, 2, 3, 0]); expectTensorsClose(outputs, expectedOutput); }); @@ -38,7 +48,7 @@ describeMathCPUAndGPU('Tests for preprocessing utils', () => { it('Peforms tfIdf encoding correctly', () => { const inputs = tensor([0, 1, 1, 2, 2, 2]); const idfWeights = tensor1d([0.1, 1.0, 10.0, 0]); - const outputs = utils.encodeCategoricalInputs(inputs, utils.tfIdf, + const outputs = utils.encodeCategoricalInputs(inputs, 'tfIdf', 4, idfWeights) as Tensor; const expectedOutput = tensor([0.1, 2, 30, 0]); expectTensorsClose(outputs, expectedOutput); @@ -46,15 +56,15 @@ describeMathCPUAndGPU('Tests for preprocessing utils', () => { it('Thows an error if input rank > 2', () => { const inputs = tensor([[[1], [2]], [[3], [1]]]); - expect(() =>utils.encodeCategoricalInputs(inputs, utils.multiHot, 4)) + expect(() =>utils.encodeCategoricalInputs(inputs, 'multiHot', 4)) .toThrowError(`When outputMode is not 'int', maximum output rank is 2 - Received outputMode ${utils.multiHot} and input shape ${inputs.shape} + Received outputMode 'multiHot' and input shape ${inputs.shape} which would result in output rank ${inputs.rank}.`); }); it('Thows an error if weights are not supplied for tfIdf', () => { const inputs = tensor([0, 1, 1, 2, 2, 2]); - expect(() =>utils.encodeCategoricalInputs(inputs, utils.tfIdf, 4)). + expect(() =>utils.encodeCategoricalInputs(inputs, 'tfIdf', 4)). toThrowError(`When outputMode is 'tfIdf', weights must be provided.`); }); diff --git a/tfjs-layers/yarn.lock b/tfjs-layers/yarn.lock index 7af3dd8ec8..0f755c13e9 100644 --- a/tfjs-layers/yarn.lock +++ b/tfjs-layers/yarn.lock @@ -7,6 +7,45 @@ resolved "https://registry.yarnpkg.com/@bazel/bazelisk/-/bazelisk-1.12.0.tgz#f08aebbf4afcb12684422450b0845dd6ef5cfe50" integrity sha512-7oQusq1e4AIyFgotxVV7Pc40Et0QyvoVjujL+7/qV5Vrbfh0Nj3CfqSgl63weEyI4r0+K6RlGVsjfRuBi05p5w== +"@tensorflow/tfjs-core@^3.20.0": + version "3.20.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.20.0.tgz#b6f89ae6490099e2c0c992faa59c96f563f9eba2" + integrity sha512-L16JyVA4a8jFJXFgB9/oYZxcGq/GfLypt5dMVTyedznARZZ9SiY/UMMbo3IKl9ZylG1dOVVTpjzV3EvBYfeJXw== + dependencies: + "@types/long" "^4.0.1" + "@types/offscreencanvas" "~2019.3.0" + "@types/seedrandom" "^2.4.28" + "@types/webgl-ext" "0.0.30" + "@webgpu/types" "0.1.16" + long "4.0.0" + node-fetch "~2.6.1" + seedrandom "^3.0.5" + +"@types/long@^4.0.1": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + +"@types/offscreencanvas@~2019.3.0": + version "2019.3.0" + resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553" + integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q== + +"@types/seedrandom@^2.4.28": + version "2.4.30" + resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.30.tgz#d2efe425869b84163c2d56e779dddadb9372cbfa" + integrity sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ== + +"@types/webgl-ext@0.0.30": + version "0.0.30" + resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d" + integrity sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg== + +"@webgpu/types@0.1.16": + version "0.1.16" + resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.16.tgz#1f05497b95b7c013facf7035c8e21784645f5cc4" + integrity sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A== + async@^3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" @@ -88,6 +127,11 @@ is-core-module@^2.2.0: dependencies: has "^1.0.3" +long@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -95,6 +139,13 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" +node-fetch@~2.6.1: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -120,6 +171,29 @@ resolve@^1.1.6: is-core-module "^2.2.0" path-parse "^1.0.6" +seedrandom@^3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7" + integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg== + +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From abd017f6bfe6dbc1d20d8fffe3143dd7a6ba81d9 Mon Sep 17 00:00:00 2001 From: AdamLang96 <45542095+AdamLang96@users.noreply.github.com> Date: Mon, 3 Oct 2022 12:53:05 -0700 Subject: [PATCH 2/5] fixed tests and reviewer reccomendations --- tfjs-layers/src/layers/preprocessing/category_encoding.ts | 3 +-- .../src/layers/preprocessing/category_encoding_test.ts | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tfjs-layers/src/layers/preprocessing/category_encoding.ts b/tfjs-layers/src/layers/preprocessing/category_encoding.ts index c6ca0f0040..f365b66898 100644 --- a/tfjs-layers/src/layers/preprocessing/category_encoding.ts +++ b/tfjs-layers/src/layers/preprocessing/category_encoding.ts @@ -19,7 +19,6 @@ import * as K from '../../backend/tfjs_backend'; import * as utils from './preprocessing_utils'; import { OutputMode } from './preprocessing_utils'; - export declare interface CategoryEncodingArgs extends LayerArgs { numTokens: number; outputMode?: OutputMode; @@ -106,7 +105,7 @@ export class CategoryEncoding extends Layer { throw new ValueError( `Input values must be between 0 < values <= numTokens - with numTokens=${this.numTokens}`); + with numTokens=${this.numTokens}`); } return utils.encodeCategoricalInputs(inputs, diff --git a/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts b/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts index c03d7076b8..39bc4bd804 100644 --- a/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts +++ b/tfjs-layers/src/layers/preprocessing/category_encoding_test.ts @@ -12,7 +12,6 @@ import { describeMathCPUAndGPU, expectTensorsClose} from '../../utils/test_utils import { Tensor, tensor} from '@tensorflow/tfjs-core'; import { CategoryEncoding } from './category_encoding'; - describeMathCPUAndGPU('Category Encoding', () => { it('Calculates correct output for Count outputMode rank 0', () => { @@ -137,7 +136,8 @@ describeMathCPUAndGPU('Category Encoding', () => { const encodingLayer = new CategoryEncoding({numTokens, outputMode: 'multiHot'}); expect(() => encodingLayer.apply(categoryData)) - .toThrowError(`Input values must be between 0 < values <= numTokens`); + .toThrowError(`Input values must be between 0 < values <= numTokens + with numTokens=${numTokens}`); }); it('Raises Value Error if min input value < 0', () => { @@ -146,6 +146,7 @@ describeMathCPUAndGPU('Category Encoding', () => { const encodingLayer = new CategoryEncoding({numTokens, outputMode: 'multiHot'}); expect(() => encodingLayer.apply(categoryData)) - .toThrowError(`Input values must be between 0 < values <= numTokens`); + .toThrowError(`Input values must be between 0 < values <= numTokens + with numTokens=${numTokens}`); }); }); From 16e0d8ad32425518e533f91fc8b3dda41bcb554c Mon Sep 17 00:00:00 2001 From: AdamLang96 <45542095+AdamLang96@users.noreply.github.com> Date: Mon, 3 Oct 2022 12:56:37 -0700 Subject: [PATCH 3/5] fixed changed files for clean pr --- tfjs-layers/package.json | 3 -- tfjs-layers/yarn.lock | 74 ---------------------------------------- 2 files changed, 77 deletions(-) diff --git a/tfjs-layers/package.json b/tfjs-layers/package.json index 31930a4b83..73c0c0d1c9 100644 --- a/tfjs-layers/package.json +++ b/tfjs-layers/package.json @@ -35,8 +35,5 @@ }, "peerDependencies": { "@tensorflow/tfjs-core": "link:../link-package/node_modules/@tensorflow/tfjs-core" - }, - "dependencies": { - "@tensorflow/tfjs-core": "^3.20.0" } } diff --git a/tfjs-layers/yarn.lock b/tfjs-layers/yarn.lock index 0f755c13e9..7af3dd8ec8 100644 --- a/tfjs-layers/yarn.lock +++ b/tfjs-layers/yarn.lock @@ -7,45 +7,6 @@ resolved "https://registry.yarnpkg.com/@bazel/bazelisk/-/bazelisk-1.12.0.tgz#f08aebbf4afcb12684422450b0845dd6ef5cfe50" integrity sha512-7oQusq1e4AIyFgotxVV7Pc40Et0QyvoVjujL+7/qV5Vrbfh0Nj3CfqSgl63weEyI4r0+K6RlGVsjfRuBi05p5w== -"@tensorflow/tfjs-core@^3.20.0": - version "3.20.0" - resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.20.0.tgz#b6f89ae6490099e2c0c992faa59c96f563f9eba2" - integrity sha512-L16JyVA4a8jFJXFgB9/oYZxcGq/GfLypt5dMVTyedznARZZ9SiY/UMMbo3IKl9ZylG1dOVVTpjzV3EvBYfeJXw== - dependencies: - "@types/long" "^4.0.1" - "@types/offscreencanvas" "~2019.3.0" - "@types/seedrandom" "^2.4.28" - "@types/webgl-ext" "0.0.30" - "@webgpu/types" "0.1.16" - long "4.0.0" - node-fetch "~2.6.1" - seedrandom "^3.0.5" - -"@types/long@^4.0.1": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" - integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== - -"@types/offscreencanvas@~2019.3.0": - version "2019.3.0" - resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553" - integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q== - -"@types/seedrandom@^2.4.28": - version "2.4.30" - resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.30.tgz#d2efe425869b84163c2d56e779dddadb9372cbfa" - integrity sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ== - -"@types/webgl-ext@0.0.30": - version "0.0.30" - resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d" - integrity sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg== - -"@webgpu/types@0.1.16": - version "0.1.16" - resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.16.tgz#1f05497b95b7c013facf7035c8e21784645f5cc4" - integrity sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A== - async@^3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" @@ -127,11 +88,6 @@ is-core-module@^2.2.0: dependencies: has "^1.0.3" -long@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" - integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== - minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -139,13 +95,6 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -node-fetch@~2.6.1: - version "2.6.7" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" - integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== - dependencies: - whatwg-url "^5.0.0" - once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -171,29 +120,6 @@ resolve@^1.1.6: is-core-module "^2.2.0" path-parse "^1.0.6" -seedrandom@^3.0.5: - version "3.0.5" - resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7" - integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg== - -tr46@~0.0.3: - version "0.0.3" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" - integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== - -webidl-conversions@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" - integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== - -whatwg-url@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" - integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== - dependencies: - tr46 "~0.0.3" - webidl-conversions "^3.0.0" - wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From bf6037a68cc97b708003319b7fbb7b4912a49f04 Mon Sep 17 00:00:00 2001 From: AdamLang96 <45542095+AdamLang96@users.noreply.github.com> Date: Mon, 3 Oct 2022 13:09:51 -0700 Subject: [PATCH 4/5] fixed indentation and linting error --- .../layers/preprocessing/category_encoding.ts | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tfjs-layers/src/layers/preprocessing/category_encoding.ts b/tfjs-layers/src/layers/preprocessing/category_encoding.ts index f365b66898..9b5eed5a6f 100644 --- a/tfjs-layers/src/layers/preprocessing/category_encoding.ts +++ b/tfjs-layers/src/layers/preprocessing/category_encoding.ts @@ -71,45 +71,45 @@ export class CategoryEncoding extends Layer { call(inputs: Tensor|Tensor[], kwargs: Kwargs): Tensor[]|Tensor { return tidy(() => { - inputs = getExactlyOneTensor(inputs); - if(inputs.dtype !== 'int32') { - inputs = K.cast(inputs, 'int32'); - } - - let countWeights: Tensor1D | Tensor2D; + inputs = getExactlyOneTensor(inputs); + if(inputs.dtype !== 'int32') { + inputs = K.cast(inputs, 'int32'); + } - if((typeof kwargs['countWeights']) !== 'undefined') { + let countWeights: Tensor1D | Tensor2D; - if(this.outputMode !== 'count') { - throw new ValueError( - `countWeights is not used when outputMode !== count. - Received countWeights=${kwargs['countWeights']}`); - } - const countWeightsRanked = getExactlyOneTensor(kwargs['countWeights']); + if((typeof kwargs['countWeights']) !== 'undefined') { - if(countWeightsRanked.rank === 1) { - countWeights = countWeightsRanked as Tensor1D; - } if(countWeightsRanked.rank === 2) { - countWeights = countWeightsRanked as Tensor2D; + if(this.outputMode !== 'count') { + throw new ValueError( + `countWeights is not used when outputMode !== count. + Received countWeights=${kwargs['countWeights']}`); } - } + const countWeightsArg = getExactlyOneTensor(kwargs['countWeights']); - const maxValue = max(inputs); - const minValue = min(inputs); - const greaterEqualMax = greater(this.numTokens, maxValue) - .bufferSync().get(0); + if(countWeightsArg.rank === 1) { + countWeights = countWeightsArg as Tensor1D; + } if(countWeightsArg.rank === 2) { + countWeights = countWeightsArg as Tensor2D; + } + } - const greaterMin = greaterEqual(minValue, 0).bufferSync().get(0); + const maxValue = max(inputs); + const minValue = min(inputs); + const greaterEqualMax = greater(this.numTokens, maxValue) + .bufferSync().get(0); - if(!(greaterEqualMax && greaterMin)) { + const greaterMin = greaterEqual(minValue, 0).bufferSync().get(0); - throw new ValueError( - `Input values must be between 0 < values <= numTokens - with numTokens=${this.numTokens}`); - } + if(!(greaterEqualMax && greaterMin)) { + + throw new ValueError( + `Input values must be between 0 < values <= numTokens + with numTokens=${this.numTokens}`); + } - return utils.encodeCategoricalInputs(inputs, - this.outputMode, this.numTokens, countWeights); + return utils.encodeCategoricalInputs(inputs, + this.outputMode, this.numTokens, countWeights); }); } } From d6b155aa0f978ea843266b71b1d235cdff778d6e Mon Sep 17 00:00:00 2001 From: AdamLang96 <45542095+AdamLang96@users.noreply.github.com> Date: Mon, 3 Oct 2022 13:20:34 -0700 Subject: [PATCH 5/5] fixed return statement in preprocessing function --- .../src/layers/preprocessing/preprocessing_utils.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts b/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts index 764fe086d3..dabc0fcce5 100644 --- a/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts +++ b/tfjs-layers/src/layers/preprocessing/preprocessing_utils.ts @@ -74,11 +74,11 @@ export function encodeCategoricalInputs(inputs: Tensor|Tensor[], return binCounts; } - if(weights === null || weights === undefined) { - throw new ValueError( - `When outputMode is 'tfIdf', weights must be provided.` - ); - } else { + if (weights) { return mul(binCounts, weights); + } else { + throw new ValueError( + `When outputMode is 'tfIdf', weights must be provided.` + ); } }