From af34ba2ada3d5cf3165686d5c69906ab94326020 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Fri, 25 Sep 2020 11:11:45 -0700 Subject: [PATCH 01/11] save --- tfjs-backend-cpu/run_tests.ts | 3 +- tfjs-backend-cpu/src/kernels/Unique.ts | 63 +++++ tfjs-backend-cpu/src/register_all_kernels.ts | 4 +- tfjs-core/scripts/enumerate-tests.js | 3 +- tfjs-core/src/kernel_names.ts | 3 + tfjs-core/src/ops/ops.ts | 1 + tfjs-core/src/ops/unique.ts | 65 +++++ tfjs-core/src/ops/unique_test.ts | 34 +++ .../chained_ops/register_all_chained_ops.ts | 1 + .../register_all_chained_ops_test.ts | 1 + tfjs-core/src/public/chained_ops/unique.ts | 32 +++ tfjs-core/src/tests.ts | 229 +----------------- 12 files changed, 208 insertions(+), 231 deletions(-) create mode 100644 tfjs-backend-cpu/src/kernels/Unique.ts create mode 100644 tfjs-core/src/ops/unique.ts create mode 100644 tfjs-core/src/ops/unique_test.ts create mode 100644 tfjs-core/src/public/chained_ops/unique.ts diff --git a/tfjs-backend-cpu/run_tests.ts b/tfjs-backend-cpu/run_tests.ts index 7152c2e5874..b2bfd868e6b 100644 --- a/tfjs-backend-cpu/run_tests.ts +++ b/tfjs-backend-cpu/run_tests.ts @@ -46,7 +46,8 @@ const customInclude = (testName: string) => { } // Include all other tests. - return true; + return testName.includes('jjj'); + // return true; }; setupTestFilters(TEST_FILTERS, customInclude); diff --git a/tfjs-backend-cpu/src/kernels/Unique.ts b/tfjs-backend-cpu/src/kernels/Unique.ts new file mode 100644 index 00000000000..2f797974838 --- /dev/null +++ b/tfjs-backend-cpu/src/kernels/Unique.ts @@ -0,0 +1,63 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +import {BackendValues, KernelConfig, KernelFunc, TensorInfo, TypedArray, Unique, UniqueInputs, util} from '@tensorflow/tfjs-core'; + +import {MathBackendCPU} from '../backend_cpu'; +import {assertNotComplex} from '../cpu_util'; + +export function unique(args: {inputs: UniqueInputs, backend: MathBackendCPU}): + TensorInfo[] { + const {inputs, backend} = args; + const {x} = inputs; + + assertNotComplex(x, 'unique'); + + const values = backend.data.get(x.dataId).values; + let xValues: TypedArray|string[] = values as TypedArray; + if (x.dtype === 'string') { + xValues = (values as Uint8Array[]).map(d => util.decodeString(d)); + } + + // A map from unique value to its index in outputValues. + const uniqueValues = new Map(); + const outputValues = []; + const indicies = new Int32Array(xValues.length); + for (let i = 0; i < xValues.length; i++) { + const value = xValues[i]; + if (uniqueValues.has(value)) { + indicies[i] = uniqueValues.get(value); + } else { + const uniqueIndex = uniqueValues.size; + uniqueValues.set(value, uniqueIndex); + indicies[i] = uniqueIndex; + outputValues.push(values[i]); + } + } + + return [ + backend.makeTensorInfo(x.shape, x.dtype, outputValues as BackendValues), + backend.makeTensorInfo( + [indicies.length], 'int32', indicies as BackendValues), + ]; +} + +export const uniqueConfig: KernelConfig = { + kernelName: Unique, + backendName: 'cpu', + kernelFunc: unique as {} as KernelFunc, +}; diff --git a/tfjs-backend-cpu/src/register_all_kernels.ts b/tfjs-backend-cpu/src/register_all_kernels.ts index bec5c0cd270..f4ffff7fec4 100644 --- a/tfjs-backend-cpu/src/register_all_kernels.ts +++ b/tfjs-backend-cpu/src/register_all_kernels.ts @@ -88,6 +88,7 @@ import {subConfig} from './kernels/Sub'; import {tanConfig} from './kernels/Tan'; import {tanhConfig} from './kernels/Tanh'; import {transposeConfig} from './kernels/Transpose'; +import {uniqueConfig} from './kernels/Unique'; // List all kernel configs here const kernelConfigs: KernelConfig[] = [ @@ -159,7 +160,8 @@ const kernelConfigs: KernelConfig[] = [ subConfig, tanConfig, tanhConfig, - transposeConfig + transposeConfig, + uniqueConfig, ]; for (const kernelConfig of kernelConfigs) { diff --git a/tfjs-core/scripts/enumerate-tests.js b/tfjs-core/scripts/enumerate-tests.js index 26b00019b45..c371898a7f7 100755 --- a/tfjs-core/scripts/enumerate-tests.js +++ b/tfjs-core/scripts/enumerate-tests.js @@ -56,7 +56,8 @@ function findTestFiles(dir, files) { !fs.existsSync(path.join(filePath, 'package.json'))) { files = findTestFiles(filePath, files); } else if ( - filePath.endsWith('_test.ts') && filePath !== 'src/setup_test.ts') { + filePath.endsWith('unique_test.ts') && + filePath !== 'src/setup_test.ts') { files.push(filePath.replace('src/', './').replace('.ts', '')); } }); diff --git a/tfjs-core/src/kernel_names.ts b/tfjs-core/src/kernel_names.ts index 25628ffb978..a4e97e5e627 100644 --- a/tfjs-core/src/kernel_names.ts +++ b/tfjs-core/src/kernel_names.ts @@ -746,6 +746,9 @@ export interface TransposeAttrs { perm: number[]; } +export const Unique = 'Unique'; +export type UniqueInputs = Pick; + export type UnaryInputs = Pick; export const Unpack = 'Unpack'; diff --git a/tfjs-core/src/ops/ops.ts b/tfjs-core/src/ops/ops.ts index 86c907532d9..9aab3f68276 100644 --- a/tfjs-core/src/ops/ops.ts +++ b/tfjs-core/src/ops/ops.ts @@ -185,6 +185,7 @@ export {tensor6d} from './tensor6d'; export {tile} from './tile'; export {topk} from './topk'; export {truncatedNormal} from './truncated_normal'; +export {unique} from './unique'; export {unsortedSegmentSum} from './unsorted_segment_sum'; export {unstack} from './unstack'; export {variable} from './variable'; diff --git a/tfjs-core/src/ops/unique.ts b/tfjs-core/src/ops/unique.ts new file mode 100644 index 00000000000..b8f2e62dea2 --- /dev/null +++ b/tfjs-core/src/ops/unique.ts @@ -0,0 +1,65 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +import {ENGINE} from '../engine'; +import {Unique, UniqueInputs} from '../kernel_names'; +import {Tensor} from '../tensor'; +import {NamedTensorMap} from '../tensor_types'; +import {convertToTensor} from '../tensor_util_env'; +import {TensorLike} from '../types'; + +import {op} from './operation'; + +/** + * Finds the unique elements in a 1-D tensor. + * + * It returns a tensor `values` containing all of the unique elements along the + * `axis` of the given tensor `x` in the same order that they occur along the + * `axis` in `x`; `x` does not need to be sorted. It also returns a tensor + * `indices` the same size as the number of the elements in `x` along the `axis` + * dimension. It contains the index in the unique output `values`. + * + * For now, only 1-D tensor is support, and the `axis` parameter is not used. + * Tensors with higher dimensions will be supported in UniqueV2. + * + * ```js + * const a = tf.tensor2d([1, 1, 2, 4, 4, 4, 7, 8, 8]); + * const {values, indices} = tf.unique(a); + * values.print(); + * indices.print(); + * ``` + * @param x 1-D or higher `tf.Tensor`. + * @param axis The axis of the tensor to find the unique elements. + * + * @doc {heading: 'Operations', subheading: 'Evaluation'} + */ +function unique_( + x: T|TensorLike, axis?: number): {values: T, indices: T} { + const $x = convertToTensor(x, 'x', 'unique'); + if ($x.shape.length !== 1) { + throw new Error(`unique() currenlty only supports 1-D tensor (got rank ${ + $x.shape.length})`); + } + + const inputs: UniqueInputs = {x: $x}; + const [values, indices] = + ENGINE.runKernel( + Unique, inputs as {} as NamedTensorMap, {} /* attrs */) as Tensor[]; + return {values, indices} as {values: T, indices: T}; +} + +export const unique = op({unique_}); diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts new file mode 100644 index 00000000000..eb4c8ef85e4 --- /dev/null +++ b/tfjs-core/src/ops/unique_test.ts @@ -0,0 +1,34 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +import * as tf from '../index'; +import {ALL_ENVS, describeWithFlags} from '../jasmine_util'; +import {expectArraysEqual} from '../test_util'; + +import {tensor1d} from './tensor1d'; + +describeWithFlags('unique jjj', ALL_ENVS, () => { + it('1d array with int32', async () => { + const x = tensor1d([1, 1, 2, 4, 4, 4, 7, 8, 8]); + const {values, indices} = tf.unique(x); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual(x.shape); + expectArraysEqual(await values.data(), [1, 2, 4, 7, 8]); + expectArraysEqual(await indices.data(), [0, 0, 1, 2, 2, 2, 3, 4, 4]); + }); +}); diff --git a/tfjs-core/src/public/chained_ops/register_all_chained_ops.ts b/tfjs-core/src/public/chained_ops/register_all_chained_ops.ts index 8aa5702ac7b..bb8d6aaa7ba 100644 --- a/tfjs-core/src/public/chained_ops/register_all_chained_ops.ts +++ b/tfjs-core/src/public/chained_ops/register_all_chained_ops.ts @@ -160,6 +160,7 @@ import './to_float'; import './to_int'; import './topk'; import './transpose'; +import './unique'; import './unsorted_segment_sum'; import './unstack'; import './where'; diff --git a/tfjs-core/src/public/chained_ops/register_all_chained_ops_test.ts b/tfjs-core/src/public/chained_ops/register_all_chained_ops_test.ts index 5549cfb6cab..9b6e559661d 100644 --- a/tfjs-core/src/public/chained_ops/register_all_chained_ops_test.ts +++ b/tfjs-core/src/public/chained_ops/register_all_chained_ops_test.ts @@ -169,6 +169,7 @@ const CHAINED_OPS = [ 'toInt', 'topk', 'transpose', + 'unique', 'unsortedSegmentSum', 'unstack', 'where', diff --git a/tfjs-core/src/public/chained_ops/unique.ts b/tfjs-core/src/public/chained_ops/unique.ts new file mode 100644 index 00000000000..7f31819a43e --- /dev/null +++ b/tfjs-core/src/public/chained_ops/unique.ts @@ -0,0 +1,32 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +import {unique} from '../../ops/unique'; +import {Tensor} from '../../tensor'; +import {Rank} from '../../types'; + +declare module '../../tensor' { + interface Tensor { + unique(this: T, axis?: number): {values: T, indices: T}; + } +} + +Tensor.prototype.unique = function( + this: T, axis?: number): {values: T, indices: T} { + this.throwIfDisposed(); + return unique(this, axis) as {values: T, indices: T}; +}; diff --git a/tfjs-core/src/tests.ts b/tfjs-core/src/tests.ts index afa7677655b..9c79f382c33 100644 --- a/tfjs-core/src/tests.ts +++ b/tfjs-core/src/tests.ts @@ -17,231 +17,4 @@ ///// DO NOT EDIT: This file is auto-generated by /scripts/enumerate-tests.js -import './browser_util_test'; -import './buffer_test'; -import './debug_mode_test'; -import './device_util_test'; -import './engine_test'; -import './environment_test'; -import './flags_test'; -import './globals_test'; -import './gradients_test'; -import './io/browser_files_test'; -import './io/http_test'; -import './io/indexed_db_test'; -import './io/io_utils_test'; -import './io/local_storage_test'; -import './io/model_management_test'; -import './io/passthrough_test'; -import './io/progress_test'; -import './io/router_registry_test'; -import './io/weights_loader_test'; -import './jasmine_util_test'; -import './kernel_registry_test'; -import './ops/abs_test'; -import './ops/acos_test'; -import './ops/acosh_test'; -import './ops/add_n_test'; -import './ops/add_test'; -import './ops/all_test'; -import './ops/any_test'; -import './ops/arg_max_test'; -import './ops/arg_min_test'; -import './ops/arithmetic_test'; -import './ops/asin_test'; -import './ops/asinh_test'; -import './ops/atan_test'; -import './ops/atanh_test'; -import './ops/avg_pool_3d_test'; -import './ops/avg_pool_test'; -import './ops/axis_util_test'; -import './ops/basic_lstm_cell_test'; -import './ops/batch_to_space_nd_test'; -import './ops/batchnorm_test'; -import './ops/binary_ops_test'; -import './ops/boolean_mask_test'; -import './ops/broadcast_to_test'; -import './ops/broadcast_util_test'; -import './ops/ceil_test'; -import './ops/clip_by_value_test'; -import './ops/clone_test'; -import './ops/compare_ops_test'; -import './ops/complex_ops_test'; -import './ops/concat_test'; -import './ops/concat_util_test'; -import './ops/confusion_matrix_test'; -import './ops/conv1d_test'; -import './ops/conv2d_depthwise_test'; -import './ops/conv2d_separable_test'; -import './ops/conv2d_test'; -import './ops/conv2d_transpose_test'; -import './ops/conv3d_test'; -import './ops/conv3d_transpose_test'; -import './ops/conv_util_test'; -import './ops/cos_test'; -import './ops/cosh_test'; -import './ops/cumsum_test'; -import './ops/depth_to_space_test'; -import './ops/diag_test'; -import './ops/dilation2d_test'; -import './ops/dropout_test'; -import './ops/dropout_util_test'; -import './ops/elu_test'; -import './ops/equal_test'; -import './ops/erf_test'; -import './ops/exp_test'; -import './ops/expand_dims_test'; -import './ops/expm1_test'; -import './ops/eye_test'; -import './ops/fill_test'; -import './ops/floor_test'; -import './ops/from_pixels_test'; -import './ops/fused/fused_conv2d_test'; -import './ops/fused/fused_depthwise_conv2d_test'; -import './ops/fused/fused_mat_mul_test'; -import './ops/gather_nd_test'; -import './ops/gather_test'; -import './ops/greater_equal_test'; -import './ops/greater_test'; -import './ops/ifft_test'; -import './ops/image/crop_and_resize_test'; -import './ops/image/flip_left_right_test'; -import './ops/image/non_max_suppression_async_test'; -import './ops/image/non_max_suppression_test'; -import './ops/image/resize_bilinear_test'; -import './ops/image/resize_nearest_neighbor_test'; -import './ops/image/rotate_with_offset_test'; -import './ops/in_top_k_test'; -import './ops/is_finite_test'; -import './ops/is_inf_test'; -import './ops/is_nan_test'; -import './ops/leaky_relu_test'; -import './ops/less_equal_test'; -import './ops/less_test'; -import './ops/linalg/band_part_test'; -import './ops/linalg/gram_schmidt_test'; -import './ops/linalg/qr_test'; -import './ops/linspace_test'; -import './ops/local_response_normalization_test'; -import './ops/log1p_test'; -import './ops/log_sigmoid_test'; -import './ops/log_softmax_test'; -import './ops/log_sum_exp_test'; -import './ops/log_test'; -import './ops/logical_and_test'; -import './ops/logical_not_test'; -import './ops/logical_or_test'; -import './ops/logical_xor_test'; -import './ops/losses/absolute_difference_test'; -import './ops/losses/compute_weighted_loss_test'; -import './ops/losses/cosine_distance_test'; -import './ops/losses/hinge_loss_test'; -import './ops/losses/huber_loss_test'; -import './ops/losses/log_loss_test'; -import './ops/losses/mean_squared_error_test'; -import './ops/losses/sigmoid_cross_entropy_test'; -import './ops/losses/softmax_cross_entropy_test'; -import './ops/mat_mul_test'; -import './ops/max_pool_3d_test'; -import './ops/max_pool_test'; -import './ops/max_pool_with_argmax_test'; -import './ops/max_test'; -import './ops/mean_test'; -import './ops/min_test'; -import './ops/moments_test'; -import './ops/moving_average_test'; -import './ops/multi_rnn_cell_test'; -import './ops/multinomial_test'; -import './ops/neg_test'; -import './ops/norm_test'; -import './ops/not_equal_test'; -import './ops/one_hot_test'; -import './ops/ones_like_test'; -import './ops/ones_test'; -import './ops/operation_test'; -import './ops/pad_test'; -import './ops/pool_test'; -import './ops/prod_test'; -import './ops/rand_test'; -import './ops/random_gamma_test'; -import './ops/random_normal_test'; -import './ops/random_uniform_test'; -import './ops/range_test'; -import './ops/reciprocal_test'; -import './ops/relu6_test'; -import './ops/relu_test'; -import './ops/reverse_1d_test'; -import './ops/reverse_2d_test'; -import './ops/reverse_3d_test'; -import './ops/reverse_4d_test'; -import './ops/reverse_test'; -import './ops/round_test'; -import './ops/rsqrt_test'; -import './ops/scatter_nd_test'; -import './ops/selu_test'; -import './ops/setdiff1d_async_test'; -import './ops/sigmoid_test'; -import './ops/sign_test'; -import './ops/signal/frame_test'; -import './ops/signal/hamming_window_test'; -import './ops/signal/hann_window_test'; -import './ops/signal/stft_test'; -import './ops/sin_test'; -import './ops/sinh_test'; -import './ops/slice1d_test'; -import './ops/slice2d_test'; -import './ops/slice3d_test'; -import './ops/slice4d_test'; -import './ops/slice_test'; -import './ops/slice_util_test'; -import './ops/softmax_test'; -import './ops/softplus_test'; -import './ops/space_to_batch_nd_test'; -import './ops/sparse_to_dense_test'; -import './ops/spectral/fft_test'; -import './ops/spectral/irfft_test'; -import './ops/spectral/rfft_test'; -import './ops/split_test'; -import './ops/sqrt_test'; -import './ops/square_test'; -import './ops/stack_test'; -import './ops/step_test'; -import './ops/strided_slice_test'; -import './ops/sub_test'; -import './ops/sum_test'; -import './ops/tan_test'; -import './ops/tanh_test'; -import './ops/tile_test'; -import './ops/to_pixels_test'; -import './ops/topk_test'; -import './ops/transpose_test'; -import './ops/truncated_normal_test'; -import './ops/unsorted_segment_sum_test'; -import './ops/unstack_test'; -import './ops/where_async_test'; -import './ops/where_test'; -import './ops/zeros_like_test'; -import './ops/zeros_test'; -import './optimizers/adadelta_optimizer_test'; -import './optimizers/adagrad_optimizer_test'; -import './optimizers/adam_optimizer_test'; -import './optimizers/adamax_optimizer_test'; -import './optimizers/momentum_optimizer_test'; -import './optimizers/optimizer_test'; -import './optimizers/rmsprop_optimizer_test'; -import './optimizers/sgd_optimizer_test'; -import './platforms/platform_browser_test'; -import './platforms/platform_node_test'; -import './profiler_test'; -import './public/chained_ops/register_all_chained_ops_test'; -import './serialization_test'; -import './tape_test'; -import './tensor_test'; -import './tensor_util_test'; -import './test_util_test'; -import './types_test'; -import './util_test'; -import './variable_test'; -import './version_test'; -import './worker_node_test'; -import './worker_test'; +import './ops/unique_test'; From d8f848042f703f701e0bd602ece24a6be199d33f Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Mon, 28 Sep 2020 09:27:23 -0700 Subject: [PATCH 02/11] save --- tfjs-backend-cpu/run_tests.ts | 3 +- tfjs-backend-cpu/src/kernels/Unique.ts | 34 +-- tfjs-backend-webgl/src/backend_webgl.ts | 5 +- tfjs-backend-webgl/src/kernels/Unique.ts | 43 ++++ .../src/register_all_kernels.ts | 4 +- tfjs-converter/docs/supported_ops.md | 1 + .../tensorflowjs/op_list/evaluation.json | 13 +- .../executors/evaluation_executor.ts | 5 + .../executors/evaluation_executor_test.ts | 10 + .../src/operations/op_list/evaluation.ts | 27 ++- tfjs-core/scripts/enumerate-tests.js | 3 +- tfjs-core/src/backends/kernel_impls.ts | 1 + tfjs-core/src/backends/unique_impl.ts | 54 +++++ tfjs-core/src/ops/unique.ts | 8 +- tfjs-core/src/ops/unique_test.ts | 44 +++- tfjs-core/src/tests.ts | 228 ++++++++++++++++++ 16 files changed, 432 insertions(+), 51 deletions(-) create mode 100644 tfjs-backend-webgl/src/kernels/Unique.ts create mode 100644 tfjs-core/src/backends/unique_impl.ts diff --git a/tfjs-backend-cpu/run_tests.ts b/tfjs-backend-cpu/run_tests.ts index b2bfd868e6b..7152c2e5874 100644 --- a/tfjs-backend-cpu/run_tests.ts +++ b/tfjs-backend-cpu/run_tests.ts @@ -46,8 +46,7 @@ const customInclude = (testName: string) => { } // Include all other tests. - return testName.includes('jjj'); - // return true; + return true; }; setupTestFilters(TEST_FILTERS, customInclude); diff --git a/tfjs-backend-cpu/src/kernels/Unique.ts b/tfjs-backend-cpu/src/kernels/Unique.ts index 2f797974838..3e777c2e24c 100644 --- a/tfjs-backend-cpu/src/kernels/Unique.ts +++ b/tfjs-backend-cpu/src/kernels/Unique.ts @@ -15,7 +15,7 @@ * ============================================================================= */ -import {BackendValues, KernelConfig, KernelFunc, TensorInfo, TypedArray, Unique, UniqueInputs, util} from '@tensorflow/tfjs-core'; +import {kernel_impls, KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; import {MathBackendCPU} from '../backend_cpu'; import {assertNotComplex} from '../cpu_util'; @@ -24,35 +24,17 @@ export function unique(args: {inputs: UniqueInputs, backend: MathBackendCPU}): TensorInfo[] { const {inputs, backend} = args; const {x} = inputs; - + if (x.shape.length !== 1) { + throw new Error(`unique() currently only supports 1-D tensor (got rank ${ + x.shape.length})`); + } assertNotComplex(x, 'unique'); const values = backend.data.get(x.dataId).values; - let xValues: TypedArray|string[] = values as TypedArray; - if (x.dtype === 'string') { - xValues = (values as Uint8Array[]).map(d => util.decodeString(d)); - } - - // A map from unique value to its index in outputValues. - const uniqueValues = new Map(); - const outputValues = []; - const indicies = new Int32Array(xValues.length); - for (let i = 0; i < xValues.length; i++) { - const value = xValues[i]; - if (uniqueValues.has(value)) { - indicies[i] = uniqueValues.get(value); - } else { - const uniqueIndex = uniqueValues.size; - uniqueValues.set(value, uniqueIndex); - indicies[i] = uniqueIndex; - outputValues.push(values[i]); - } - } - + const {outputValues, indices} = kernel_impls.uniqueImpl(values, x.dtype); return [ - backend.makeTensorInfo(x.shape, x.dtype, outputValues as BackendValues), - backend.makeTensorInfo( - [indicies.length], 'int32', indicies as BackendValues), + backend.makeTensorInfo(x.shape, x.dtype, outputValues), + backend.makeTensorInfo([indices.length], 'int32', indices), ]; } diff --git a/tfjs-backend-webgl/src/backend_webgl.ts b/tfjs-backend-webgl/src/backend_webgl.ts index d6a41def911..d07bfcb6930 100644 --- a/tfjs-backend-webgl/src/backend_webgl.ts +++ b/tfjs-backend-webgl/src/backend_webgl.ts @@ -2360,8 +2360,9 @@ export class MathBackendWebGL extends KernelBackend { return backend_util.linspaceImpl(start, stop, num); } - makeTensorInfo(shape: number[], dtype: DataType): TensorInfo { - const dataId = this.write(null /* values */, shape, dtype); + makeTensorInfo(shape: number[], dtype: DataType, values?: BackendValues): + TensorInfo { + const dataId = this.write(values, shape, dtype); this.texData.get(dataId).usage = null; return {dataId, shape, dtype}; } diff --git a/tfjs-backend-webgl/src/kernels/Unique.ts b/tfjs-backend-webgl/src/kernels/Unique.ts new file mode 100644 index 00000000000..aaeec4269ec --- /dev/null +++ b/tfjs-backend-webgl/src/kernels/Unique.ts @@ -0,0 +1,43 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +import {kernel_impls, KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; +import {MathBackendWebGL} from '../backend_webgl'; + +export function unique(args: {inputs: UniqueInputs, backend: MathBackendWebGL}): + TensorInfo[] { + const {inputs, backend} = args; + const {x} = inputs; + if (x.shape.length !== 1) { + throw new Error(`unique() currently only supports 1-D tensor (got rank ${ + x.shape.length})`); + } + + // Download data and use the shared cpu implementation. + const values = backend.readSync(x.dataId); + const {outputValues, indices} = kernel_impls.uniqueImpl(values, x.dtype); + return [ + backend.makeTensorInfo(x.shape, x.dtype, outputValues), + backend.makeTensorInfo([indices.length], 'int32', indices), + ]; +} + +export const uniqueConfig: KernelConfig = { + kernelName: Unique, + backendName: 'webgl', + kernelFunc: unique as {} as KernelFunc, +}; diff --git a/tfjs-backend-webgl/src/register_all_kernels.ts b/tfjs-backend-webgl/src/register_all_kernels.ts index 48161cc1f2b..a94a871145e 100644 --- a/tfjs-backend-webgl/src/register_all_kernels.ts +++ b/tfjs-backend-webgl/src/register_all_kernels.ts @@ -39,6 +39,7 @@ import {squareConfig} from './kernels/Square'; import {squaredDifferenceConfig} from './kernels/SquaredDifference'; import {tanConfig} from './kernels/Tan'; import {transposeConfig} from './kernels/Transpose'; +import {uniqueConfig} from './kernels/Unique'; // List all kernel configs here const kernelConfigs: KernelConfig[] = [ @@ -64,7 +65,8 @@ const kernelConfigs: KernelConfig[] = [ squareConfig, squaredDifferenceConfig, tanConfig, - transposeConfig + transposeConfig, + uniqueConfig, ]; for (const kernelConfig of kernelConfigs) { diff --git a/tfjs-converter/docs/supported_ops.md b/tfjs-converter/docs/supported_ops.md index 9db33accd58..82c80045e25 100644 --- a/tfjs-converter/docs/supported_ops.md +++ b/tfjs-converter/docs/supported_ops.md @@ -166,6 +166,7 @@ |Tensorflow Op Name|Tensorflow.js Op Name| |---|---| |TopKV2|topK| +|Unique|unique| |Not mapped|confusionMatrix| |Not mapped|topk| diff --git a/tfjs-converter/python/tensorflowjs/op_list/evaluation.json b/tfjs-converter/python/tensorflowjs/op_list/evaluation.json index 8a14458a4ed..c0b88c07183 100644 --- a/tfjs-converter/python/tensorflowjs/op_list/evaluation.json +++ b/tfjs-converter/python/tensorflowjs/op_list/evaluation.json @@ -21,5 +21,16 @@ "type": "bool" } ] + }, + { + "tfOpName": "Unique", + "category": "evaluation", + "inputs": [ + { + "start": 0, + "name": "x", + "type": "tensor" + } + ] } -] \ No newline at end of file +] diff --git a/tfjs-converter/src/operations/executors/evaluation_executor.ts b/tfjs-converter/src/operations/executors/evaluation_executor.ts index 84ad0cf5601..157ad1e1861 100644 --- a/tfjs-converter/src/operations/executors/evaluation_executor.ts +++ b/tfjs-converter/src/operations/executors/evaluation_executor.ts @@ -37,6 +37,11 @@ export const executeOp: InternalOpExecutor = const result = tfOps.topk(x, k, sorted); return [result.values, result.indices]; } + case 'Unique': { + const x = getParamValue('x', node, tensorMap, context) as Tensor; + const result = tfOps.unique(x); + return [result.values, result.indices]; + } default: throw TypeError(`Node type ${node.op} is not implemented`); } diff --git a/tfjs-converter/src/operations/executors/evaluation_executor_test.ts b/tfjs-converter/src/operations/executors/evaluation_executor_test.ts index 13e7cfec569..ede54c2a97b 100644 --- a/tfjs-converter/src/operations/executors/evaluation_executor_test.ts +++ b/tfjs-converter/src/operations/executors/evaluation_executor_test.ts @@ -54,5 +54,15 @@ describe('evaluation', () => { expect(tfOps.topk).toHaveBeenCalledWith(input1[0], 1, true); }); }); + + describe('Unique', () => { + it('should return input', () => { + node.op = 'Unique'; + node.inputParams['x'] = createTensorAttr(0); + spyOn(tfOps, 'unique').and.callThrough(); + executeOp(node, {input1}, context); + expect(tfOps.unique).toHaveBeenCalledWith(input1[0]); + }); + }); }); }); diff --git a/tfjs-converter/src/operations/op_list/evaluation.ts b/tfjs-converter/src/operations/op_list/evaluation.ts index 5e80b493969..9d631a94de0 100644 --- a/tfjs-converter/src/operations/op_list/evaluation.ts +++ b/tfjs-converter/src/operations/op_list/evaluation.ts @@ -17,12 +17,21 @@ import {OpMapper} from '../types'; * ============================================================================= */ -export const json: OpMapper[] = [{ - 'tfOpName': 'TopKV2', - 'category': 'evaluation', - 'inputs': [ - {'start': 0, 'name': 'x', 'type': 'tensor'}, - {'start': 1, 'name': 'k', 'type': 'number'}, - ], - 'attrs': [{'tfName': 'sorted', 'name': 'sorted', 'type': 'bool'}] -}]; +export const json: OpMapper[] = [ + { + 'tfOpName': 'TopKV2', + 'category': 'evaluation', + 'inputs': [ + {'start': 0, 'name': 'x', 'type': 'tensor'}, + {'start': 1, 'name': 'k', 'type': 'number'}, + ], + 'attrs': [{'tfName': 'sorted', 'name': 'sorted', 'type': 'bool'}] + }, + { + 'tfOpName': 'Unique', + 'category': 'evaluation', + 'inputs': [ + {'start': 0, 'name': 'x', 'type': 'tensor'}, + ], + }, +]; diff --git a/tfjs-core/scripts/enumerate-tests.js b/tfjs-core/scripts/enumerate-tests.js index c371898a7f7..26b00019b45 100755 --- a/tfjs-core/scripts/enumerate-tests.js +++ b/tfjs-core/scripts/enumerate-tests.js @@ -56,8 +56,7 @@ function findTestFiles(dir, files) { !fs.existsSync(path.join(filePath, 'package.json'))) { files = findTestFiles(filePath, files); } else if ( - filePath.endsWith('unique_test.ts') && - filePath !== 'src/setup_test.ts') { + filePath.endsWith('_test.ts') && filePath !== 'src/setup_test.ts') { files.push(filePath.replace('src/', './').replace('.ts', '')); } }); diff --git a/tfjs-core/src/backends/kernel_impls.ts b/tfjs-core/src/backends/kernel_impls.ts index 093f1f7bd0d..78d785573ad 100644 --- a/tfjs-core/src/backends/kernel_impls.ts +++ b/tfjs-core/src/backends/kernel_impls.ts @@ -19,4 +19,5 @@ export {nonMaxSuppressionV3Impl, nonMaxSuppressionV4Impl, nonMaxSuppressionV5Imp export {split} from './split_shared'; export {tile} from './tile_impl'; export {topkImpl} from './topk_impl'; +export {uniqueImpl} from './unique_impl'; export {whereImpl} from './where_impl'; diff --git a/tfjs-core/src/backends/unique_impl.ts b/tfjs-core/src/backends/unique_impl.ts new file mode 100644 index 00000000000..41cbd51071e --- /dev/null +++ b/tfjs-core/src/backends/unique_impl.ts @@ -0,0 +1,54 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * An implementation of the unique kernel shared between webgl and cpu. + */ + +import {BackendValues, DataType, TypedArray} from '../types'; +import {decodeString} from '../util'; + +export function uniqueImpl(values: BackendValues, dtype: DataType): + {outputValues: BackendValues, indices: BackendValues} { + let xValues: TypedArray|string[] = []; + if (dtype === 'string') { + xValues = (values as Uint8Array[]).map(d => decodeString(d)); + } else { + xValues = values as TypedArray; + } + + // A map from unique value to its index in outputValues. + const uniqueValues = new Map(); + const outputValues = []; + const indices = new Int32Array(xValues.length); + for (let i = 0; i < xValues.length; i++) { + const value = xValues[i]; + if (uniqueValues.has(value)) { + indices[i] = uniqueValues.get(value); + } else { + const uniqueIndex = uniqueValues.size; + uniqueValues.set(value, uniqueIndex); + indices[i] = uniqueIndex; + outputValues.push(values[i]); + } + } + + return { + outputValues: outputValues as BackendValues, + indices, + }; +} diff --git a/tfjs-core/src/ops/unique.ts b/tfjs-core/src/ops/unique.ts index b8f2e62dea2..7f698143ab6 100644 --- a/tfjs-core/src/ops/unique.ts +++ b/tfjs-core/src/ops/unique.ts @@ -49,12 +49,8 @@ import {op} from './operation'; */ function unique_( x: T|TensorLike, axis?: number): {values: T, indices: T} { - const $x = convertToTensor(x, 'x', 'unique'); - if ($x.shape.length !== 1) { - throw new Error(`unique() currenlty only supports 1-D tensor (got rank ${ - $x.shape.length})`); - } - + // x can be of any dtype, thus null as the last argument. + const $x = convertToTensor(x, 'x', 'unique', null); const inputs: UniqueInputs = {x: $x}; const [values, indices] = ENGINE.runKernel( diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts index eb4c8ef85e4..6751f2002b8 100644 --- a/tfjs-core/src/ops/unique_test.ts +++ b/tfjs-core/src/ops/unique_test.ts @@ -21,8 +21,8 @@ import {expectArraysEqual} from '../test_util'; import {tensor1d} from './tensor1d'; -describeWithFlags('unique jjj', ALL_ENVS, () => { - it('1d array with int32', async () => { +describeWithFlags('unique', ALL_ENVS, () => { + it('1d tensor with int32', async () => { const x = tensor1d([1, 1, 2, 4, 4, 4, 7, 8, 8]); const {values, indices} = tf.unique(x); @@ -31,4 +31,44 @@ describeWithFlags('unique jjj', ALL_ENVS, () => { expectArraysEqual(await values.data(), [1, 2, 4, 7, 8]); expectArraysEqual(await indices.data(), [0, 0, 1, 2, 2, 2, 3, 4, 4]); }); + + it('1d tensor with random int32', async () => { + const x = tf.randomNormal([700], 5, 2, 'int32'); + const {values, indices} = tf.unique(x); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual(x.shape); + const xValues = await x.data(); + const outputValues = await values.data(); + const indicesValues = await indices.data(); + for (let i = 0; i < xValues.length; i++) { + expect(xValues[i]).toBe(outputValues[indicesValues[i]]); + } + }); + + it('1d tensor with string', async () => { + const x = tensor1d(['a', 'b', 'b', 'c', 'c']); + const {values, indices} = tf.unique(x); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual(x.shape); + expectArraysEqual(await values.data(), ['a', 'b', 'c']); + expectArraysEqual(await indices.data(), [0, 1, 1, 2, 2]); + }); + + it('1d tensor with bool', async () => { + const x = tensor1d([true, true, false]); + const {values, indices} = tf.unique(x); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual(x.shape); + expectArraysEqual(await values.data(), [true, false]); + expectArraysEqual(await indices.data(), [0, 0, 1]); + }); + + it('throws for non 1-D tensor', () => { + expect(() => tf.unique([[1, 2], [3, 4]])) + .toThrowError( + /unique\(\) currently only supports 1-D tensor.*got rank 2.*/); + }); }); diff --git a/tfjs-core/src/tests.ts b/tfjs-core/src/tests.ts index 9c79f382c33..c27997d84d0 100644 --- a/tfjs-core/src/tests.ts +++ b/tfjs-core/src/tests.ts @@ -17,4 +17,232 @@ ///// DO NOT EDIT: This file is auto-generated by /scripts/enumerate-tests.js +import './browser_util_test'; +import './buffer_test'; +import './debug_mode_test'; +import './device_util_test'; +import './engine_test'; +import './environment_test'; +import './flags_test'; +import './globals_test'; +import './gradients_test'; +import './io/browser_files_test'; +import './io/http_test'; +import './io/indexed_db_test'; +import './io/io_utils_test'; +import './io/local_storage_test'; +import './io/model_management_test'; +import './io/passthrough_test'; +import './io/progress_test'; +import './io/router_registry_test'; +import './io/weights_loader_test'; +import './jasmine_util_test'; +import './kernel_registry_test'; +import './ops/abs_test'; +import './ops/acos_test'; +import './ops/acosh_test'; +import './ops/add_n_test'; +import './ops/add_test'; +import './ops/all_test'; +import './ops/any_test'; +import './ops/arg_max_test'; +import './ops/arg_min_test'; +import './ops/arithmetic_test'; +import './ops/asin_test'; +import './ops/asinh_test'; +import './ops/atan_test'; +import './ops/atanh_test'; +import './ops/avg_pool_3d_test'; +import './ops/avg_pool_test'; +import './ops/axis_util_test'; +import './ops/basic_lstm_cell_test'; +import './ops/batch_to_space_nd_test'; +import './ops/batchnorm_test'; +import './ops/binary_ops_test'; +import './ops/boolean_mask_test'; +import './ops/broadcast_to_test'; +import './ops/broadcast_util_test'; +import './ops/ceil_test'; +import './ops/clip_by_value_test'; +import './ops/clone_test'; +import './ops/compare_ops_test'; +import './ops/complex_ops_test'; +import './ops/concat_test'; +import './ops/concat_util_test'; +import './ops/confusion_matrix_test'; +import './ops/conv1d_test'; +import './ops/conv2d_depthwise_test'; +import './ops/conv2d_separable_test'; +import './ops/conv2d_test'; +import './ops/conv2d_transpose_test'; +import './ops/conv3d_test'; +import './ops/conv3d_transpose_test'; +import './ops/conv_util_test'; +import './ops/cos_test'; +import './ops/cosh_test'; +import './ops/cumsum_test'; +import './ops/depth_to_space_test'; +import './ops/diag_test'; +import './ops/dilation2d_test'; +import './ops/dropout_test'; +import './ops/dropout_util_test'; +import './ops/elu_test'; +import './ops/equal_test'; +import './ops/erf_test'; +import './ops/exp_test'; +import './ops/expand_dims_test'; +import './ops/expm1_test'; +import './ops/eye_test'; +import './ops/fill_test'; +import './ops/floor_test'; +import './ops/from_pixels_test'; +import './ops/fused/fused_conv2d_test'; +import './ops/fused/fused_depthwise_conv2d_test'; +import './ops/fused/fused_mat_mul_test'; +import './ops/gather_nd_test'; +import './ops/gather_test'; +import './ops/greater_equal_test'; +import './ops/greater_test'; +import './ops/ifft_test'; +import './ops/image/crop_and_resize_test'; +import './ops/image/flip_left_right_test'; +import './ops/image/non_max_suppression_async_test'; +import './ops/image/non_max_suppression_test'; +import './ops/image/resize_bilinear_test'; +import './ops/image/resize_nearest_neighbor_test'; +import './ops/image/rotate_with_offset_test'; +import './ops/in_top_k_test'; +import './ops/is_finite_test'; +import './ops/is_inf_test'; +import './ops/is_nan_test'; +import './ops/leaky_relu_test'; +import './ops/less_equal_test'; +import './ops/less_test'; +import './ops/linalg/band_part_test'; +import './ops/linalg/gram_schmidt_test'; +import './ops/linalg/qr_test'; +import './ops/linspace_test'; +import './ops/local_response_normalization_test'; +import './ops/log1p_test'; +import './ops/log_sigmoid_test'; +import './ops/log_softmax_test'; +import './ops/log_sum_exp_test'; +import './ops/log_test'; +import './ops/logical_and_test'; +import './ops/logical_not_test'; +import './ops/logical_or_test'; +import './ops/logical_xor_test'; +import './ops/losses/absolute_difference_test'; +import './ops/losses/compute_weighted_loss_test'; +import './ops/losses/cosine_distance_test'; +import './ops/losses/hinge_loss_test'; +import './ops/losses/huber_loss_test'; +import './ops/losses/log_loss_test'; +import './ops/losses/mean_squared_error_test'; +import './ops/losses/sigmoid_cross_entropy_test'; +import './ops/losses/softmax_cross_entropy_test'; +import './ops/mat_mul_test'; +import './ops/max_pool_3d_test'; +import './ops/max_pool_test'; +import './ops/max_pool_with_argmax_test'; +import './ops/max_test'; +import './ops/mean_test'; +import './ops/min_test'; +import './ops/moments_test'; +import './ops/moving_average_test'; +import './ops/multi_rnn_cell_test'; +import './ops/multinomial_test'; +import './ops/neg_test'; +import './ops/norm_test'; +import './ops/not_equal_test'; +import './ops/one_hot_test'; +import './ops/ones_like_test'; +import './ops/ones_test'; +import './ops/operation_test'; +import './ops/pad_test'; +import './ops/pool_test'; +import './ops/prod_test'; +import './ops/rand_test'; +import './ops/random_gamma_test'; +import './ops/random_normal_test'; +import './ops/random_uniform_test'; +import './ops/range_test'; +import './ops/reciprocal_test'; +import './ops/relu6_test'; +import './ops/relu_test'; +import './ops/reverse_1d_test'; +import './ops/reverse_2d_test'; +import './ops/reverse_3d_test'; +import './ops/reverse_4d_test'; +import './ops/reverse_test'; +import './ops/round_test'; +import './ops/rsqrt_test'; +import './ops/scatter_nd_test'; +import './ops/selu_test'; +import './ops/setdiff1d_async_test'; +import './ops/sigmoid_test'; +import './ops/sign_test'; +import './ops/signal/frame_test'; +import './ops/signal/hamming_window_test'; +import './ops/signal/hann_window_test'; +import './ops/signal/stft_test'; +import './ops/sin_test'; +import './ops/sinh_test'; +import './ops/slice1d_test'; +import './ops/slice2d_test'; +import './ops/slice3d_test'; +import './ops/slice4d_test'; +import './ops/slice_test'; +import './ops/slice_util_test'; +import './ops/softmax_test'; +import './ops/softplus_test'; +import './ops/space_to_batch_nd_test'; +import './ops/sparse_to_dense_test'; +import './ops/spectral/fft_test'; +import './ops/spectral/irfft_test'; +import './ops/spectral/rfft_test'; +import './ops/split_test'; +import './ops/sqrt_test'; +import './ops/square_test'; +import './ops/stack_test'; +import './ops/step_test'; +import './ops/strided_slice_test'; +import './ops/sub_test'; +import './ops/sum_test'; +import './ops/tan_test'; +import './ops/tanh_test'; +import './ops/tile_test'; +import './ops/to_pixels_test'; +import './ops/topk_test'; +import './ops/transpose_test'; +import './ops/truncated_normal_test'; import './ops/unique_test'; +import './ops/unsorted_segment_sum_test'; +import './ops/unstack_test'; +import './ops/where_async_test'; +import './ops/where_test'; +import './ops/zeros_like_test'; +import './ops/zeros_test'; +import './optimizers/adadelta_optimizer_test'; +import './optimizers/adagrad_optimizer_test'; +import './optimizers/adam_optimizer_test'; +import './optimizers/adamax_optimizer_test'; +import './optimizers/momentum_optimizer_test'; +import './optimizers/optimizer_test'; +import './optimizers/rmsprop_optimizer_test'; +import './optimizers/sgd_optimizer_test'; +import './platforms/platform_browser_test'; +import './platforms/platform_node_test'; +import './profiler_test'; +import './public/chained_ops/register_all_chained_ops_test'; +import './serialization_test'; +import './tape_test'; +import './tensor_test'; +import './tensor_util_test'; +import './test_util_test'; +import './types_test'; +import './util_test'; +import './variable_test'; +import './version_test'; +import './worker_node_test'; +import './worker_test'; From e28027aa8825f2d813b2da83f59ba14c167f42f6 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Thu, 1 Oct 2020 15:00:17 -0700 Subject: [PATCH 03/11] save --- tfjs-backend-cpu/src/kernels/Unique.ts | 9 ++---- .../src/kernels/Unique_impl.ts | 9 ++---- tfjs-backend-cpu/src/shared.ts | 1 + tfjs-backend-webgl/src/kernel_utils/shared.ts | 8 ++++-- tfjs-backend-webgl/src/kernels/Unique.ts | 14 +++++----- tfjs-converter/metadata/kernel2op.json | 3 ++ tfjs-core/src/backends/kernel_impls.ts | 1 - tfjs-core/src/ops/unique.ts | 28 ++++++++++++------- tfjs-core/src/ops/unique_test.ts | 14 ---------- 9 files changed, 40 insertions(+), 47 deletions(-) rename tfjs-core/src/backends/unique_impl.ts => tfjs-backend-cpu/src/kernels/Unique_impl.ts (86%) diff --git a/tfjs-backend-cpu/src/kernels/Unique.ts b/tfjs-backend-cpu/src/kernels/Unique.ts index 3e777c2e24c..69b9912753d 100644 --- a/tfjs-backend-cpu/src/kernels/Unique.ts +++ b/tfjs-backend-cpu/src/kernels/Unique.ts @@ -15,23 +15,20 @@ * ============================================================================= */ -import {kernel_impls, KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; +import {KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; import {MathBackendCPU} from '../backend_cpu'; import {assertNotComplex} from '../cpu_util'; +import {uniqueImpl} from './Unique_impl'; export function unique(args: {inputs: UniqueInputs, backend: MathBackendCPU}): TensorInfo[] { const {inputs, backend} = args; const {x} = inputs; - if (x.shape.length !== 1) { - throw new Error(`unique() currently only supports 1-D tensor (got rank ${ - x.shape.length})`); - } assertNotComplex(x, 'unique'); const values = backend.data.get(x.dataId).values; - const {outputValues, indices} = kernel_impls.uniqueImpl(values, x.dtype); + const {outputValues, indices} = uniqueImpl(values, x.dtype); return [ backend.makeTensorInfo(x.shape, x.dtype, outputValues), backend.makeTensorInfo([indices.length], 'int32', indices), diff --git a/tfjs-core/src/backends/unique_impl.ts b/tfjs-backend-cpu/src/kernels/Unique_impl.ts similarity index 86% rename from tfjs-core/src/backends/unique_impl.ts rename to tfjs-backend-cpu/src/kernels/Unique_impl.ts index 41cbd51071e..a442b8b0097 100644 --- a/tfjs-core/src/backends/unique_impl.ts +++ b/tfjs-backend-cpu/src/kernels/Unique_impl.ts @@ -15,18 +15,13 @@ * ============================================================================= */ -/** - * An implementation of the unique kernel shared between webgl and cpu. - */ - -import {BackendValues, DataType, TypedArray} from '../types'; -import {decodeString} from '../util'; +import {BackendValues, DataType, TypedArray, util} from '@tensorflow/tfjs-core'; export function uniqueImpl(values: BackendValues, dtype: DataType): {outputValues: BackendValues, indices: BackendValues} { let xValues: TypedArray|string[] = []; if (dtype === 'string') { - xValues = (values as Uint8Array[]).map(d => decodeString(d)); + xValues = (values as Uint8Array[]).map(d => util.decodeString(d)); } else { xValues = values as TypedArray; } diff --git a/tfjs-backend-cpu/src/shared.ts b/tfjs-backend-cpu/src/shared.ts index c54f9b4b7e3..bad8613065b 100644 --- a/tfjs-backend-cpu/src/shared.ts +++ b/tfjs-backend-cpu/src/shared.ts @@ -18,3 +18,4 @@ // Shared kernel impls for use in other backends. export {maxImpl} from './kernels/Max_impl'; export {transposeImpl} from './kernels/Transpose_impl'; +export {uniqueImpl} from './kernels/Unique_impl'; diff --git a/tfjs-backend-webgl/src/kernel_utils/shared.ts b/tfjs-backend-webgl/src/kernel_utils/shared.ts index 0ac71e981c0..6c2941d0e83 100644 --- a/tfjs-backend-webgl/src/kernel_utils/shared.ts +++ b/tfjs-backend-webgl/src/kernel_utils/shared.ts @@ -20,6 +20,10 @@ // tslint:disable-next-line: no-imports-from-dist import * as shared from '@tensorflow/tfjs-backend-cpu/dist/shared'; -const {maxImpl: maxImplCPU, transposeImpl: transposeImplCPU} = shared; +const { + maxImpl: maxImplCPU, + transposeImpl: transposeImplCPU, + uniqueImpl: uniqueImplCPU, +} = shared; -export {maxImplCPU, transposeImplCPU}; +export {maxImplCPU, transposeImplCPU, uniqueImplCPU}; diff --git a/tfjs-backend-webgl/src/kernels/Unique.ts b/tfjs-backend-webgl/src/kernels/Unique.ts index aaeec4269ec..62c08bc5a5d 100644 --- a/tfjs-backend-webgl/src/kernels/Unique.ts +++ b/tfjs-backend-webgl/src/kernels/Unique.ts @@ -15,21 +15,21 @@ * ============================================================================= */ -import {kernel_impls, KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; +import {KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; + import {MathBackendWebGL} from '../backend_webgl'; +import {uniqueImplCPU} from '../kernel_utils/shared'; +import {assertNotComplex} from '../webgl_util'; export function unique(args: {inputs: UniqueInputs, backend: MathBackendWebGL}): TensorInfo[] { const {inputs, backend} = args; const {x} = inputs; - if (x.shape.length !== 1) { - throw new Error(`unique() currently only supports 1-D tensor (got rank ${ - x.shape.length})`); - } + assertNotComplex(x, 'unique'); - // Download data and use the shared cpu implementation. + // For now, always forward calculation to the CPU backend. const values = backend.readSync(x.dataId); - const {outputValues, indices} = kernel_impls.uniqueImpl(values, x.dtype); + const {outputValues, indices} = uniqueImplCPU(values, x.dtype); return [ backend.makeTensorInfo(x.shape, x.dtype, outputValues), backend.makeTensorInfo([indices.length], 'int32', indices), diff --git a/tfjs-converter/metadata/kernel2op.json b/tfjs-converter/metadata/kernel2op.json index 7d7e5383749..88b359c3639 100644 --- a/tfjs-converter/metadata/kernel2op.json +++ b/tfjs-converter/metadata/kernel2op.json @@ -505,6 +505,9 @@ "TruncatedNormal": [ "truncatedNormal" ], + "Unique": [ + "unique" + ], "Unpack": [ "unstack" ], diff --git a/tfjs-core/src/backends/kernel_impls.ts b/tfjs-core/src/backends/kernel_impls.ts index 78d785573ad..093f1f7bd0d 100644 --- a/tfjs-core/src/backends/kernel_impls.ts +++ b/tfjs-core/src/backends/kernel_impls.ts @@ -19,5 +19,4 @@ export {nonMaxSuppressionV3Impl, nonMaxSuppressionV4Impl, nonMaxSuppressionV5Imp export {split} from './split_shared'; export {tile} from './tile_impl'; export {topkImpl} from './topk_impl'; -export {uniqueImpl} from './unique_impl'; export {whereImpl} from './where_impl'; diff --git a/tfjs-core/src/ops/unique.ts b/tfjs-core/src/ops/unique.ts index 7f698143ab6..5586fbffddc 100644 --- a/tfjs-core/src/ops/unique.ts +++ b/tfjs-core/src/ops/unique.ts @@ -17,10 +17,11 @@ import {ENGINE} from '../engine'; import {Unique, UniqueInputs} from '../kernel_names'; -import {Tensor} from '../tensor'; +import {Tensor, Tensor1D} from '../tensor'; import {NamedTensorMap} from '../tensor_types'; import {convertToTensor} from '../tensor_util_env'; import {TensorLike} from '../types'; +import * as util from '../util'; import {op} from './operation'; @@ -39,23 +40,30 @@ import {op} from './operation'; * ```js * const a = tf.tensor2d([1, 1, 2, 4, 4, 4, 7, 8, 8]); * const {values, indices} = tf.unique(a); - * values.print(); - * indices.print(); + * values.print(); // [1, 2, 4, 7, 8, 9] + * indices.print(); // [0, 0, 1, 2, 2, 2, 3, 4, 4] * ``` - * @param x 1-D or higher `tf.Tensor`. - * @param axis The axis of the tensor to find the unique elements. + * @param x 1-D tensor (int32, string, bool). + * @param axis The axis of the tensor to find the unique elements (not used for + * now). + * @returns [uniqueValues, indices (see above)] * * @doc {heading: 'Operations', subheading: 'Evaluation'} */ function unique_( - x: T|TensorLike, axis?: number): {values: T, indices: T} { + x: T|TensorLike, axis?: number): {values: Tensor1D, indices: Tensor1D} { // x can be of any dtype, thus null as the last argument. const $x = convertToTensor(x, 'x', 'unique', null); + util.assert( + $x.rank === 1, + () => 'unique() currently only supports 1-D tensor ' + + `(got rank ${$x.rank})`); + const inputs: UniqueInputs = {x: $x}; - const [values, indices] = - ENGINE.runKernel( - Unique, inputs as {} as NamedTensorMap, {} /* attrs */) as Tensor[]; - return {values, indices} as {values: T, indices: T}; + const [values, indices] = ENGINE.runKernel( + Unique, inputs as {} as NamedTensorMap, + {} /* attrs */) as [Tensor1D, Tensor1D]; + return {values, indices}; } export const unique = op({unique_}); diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts index 6751f2002b8..35f0d27c493 100644 --- a/tfjs-core/src/ops/unique_test.ts +++ b/tfjs-core/src/ops/unique_test.ts @@ -32,20 +32,6 @@ describeWithFlags('unique', ALL_ENVS, () => { expectArraysEqual(await indices.data(), [0, 0, 1, 2, 2, 2, 3, 4, 4]); }); - it('1d tensor with random int32', async () => { - const x = tf.randomNormal([700], 5, 2, 'int32'); - const {values, indices} = tf.unique(x); - - expect(indices.dtype).toBe('int32'); - expect(indices.shape).toEqual(x.shape); - const xValues = await x.data(); - const outputValues = await values.data(); - const indicesValues = await indices.data(); - for (let i = 0; i < xValues.length; i++) { - expect(xValues[i]).toBe(outputValues[indicesValues[i]]); - } - }); - it('1d tensor with string', async () => { const x = tensor1d(['a', 'b', 'b', 'c', 'c']); const {values, indices} = tf.unique(x); From ae6c1a5131db9f593aee9b4ee933e3d8a3089107 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Thu, 1 Oct 2020 15:07:22 -0700 Subject: [PATCH 04/11] typo --- .../src/operations/executors/evaluation_executor_test.ts | 2 +- tfjs-core/src/ops/unique.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tfjs-converter/src/operations/executors/evaluation_executor_test.ts b/tfjs-converter/src/operations/executors/evaluation_executor_test.ts index ede54c2a97b..ad7e36d5fab 100644 --- a/tfjs-converter/src/operations/executors/evaluation_executor_test.ts +++ b/tfjs-converter/src/operations/executors/evaluation_executor_test.ts @@ -56,7 +56,7 @@ describe('evaluation', () => { }); describe('Unique', () => { - it('should return input', () => { + it('should get called correctly', () => { node.op = 'Unique'; node.inputParams['x'] = createTensorAttr(0); spyOn(tfOps, 'unique').and.callThrough(); diff --git a/tfjs-core/src/ops/unique.ts b/tfjs-core/src/ops/unique.ts index 5586fbffddc..e8bb72849e5 100644 --- a/tfjs-core/src/ops/unique.ts +++ b/tfjs-core/src/ops/unique.ts @@ -34,7 +34,7 @@ import {op} from './operation'; * `indices` the same size as the number of the elements in `x` along the `axis` * dimension. It contains the index in the unique output `values`. * - * For now, only 1-D tensor is support, and the `axis` parameter is not used. + * For now, only 1-D tensor is supported, and the `axis` parameter is not used. * Tensors with higher dimensions will be supported in UniqueV2. * * ```js From bcc6845de8f9374f07a67f0127f3377255da0954 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Fri, 2 Oct 2020 16:02:58 -0700 Subject: [PATCH 05/11] v2 --- tfjs-backend-cpu/src/kernels/Unique.ts | 14 ++- tfjs-backend-cpu/src/kernels/Unique_impl.ts | 111 ++++++++++++++---- tfjs-backend-webgl/src/kernels/Unique.ts | 16 ++- tfjs-converter/metadata/kernel2op.json | 3 + .../tensorflowjs/op_list/evaluation.json | 16 +++ .../executors/evaluation_executor.ts | 7 ++ .../executors/evaluation_executor_test.ts | 13 ++ .../src/operations/op_list/evaluation.ts | 8 ++ tfjs-core/src/kernel_names.ts | 3 + tfjs-core/src/ops/unique.ts | 54 ++++++--- tfjs-core/src/ops/unique_test.ts | 22 +++- 11 files changed, 213 insertions(+), 54 deletions(-) diff --git a/tfjs-backend-cpu/src/kernels/Unique.ts b/tfjs-backend-cpu/src/kernels/Unique.ts index 69b9912753d..f4b4d23b31a 100644 --- a/tfjs-backend-cpu/src/kernels/Unique.ts +++ b/tfjs-backend-cpu/src/kernels/Unique.ts @@ -15,22 +15,26 @@ * ============================================================================= */ -import {KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; +import {KernelConfig, KernelFunc, TensorInfo, Unique, UniqueAttrs, UniqueInputs} from '@tensorflow/tfjs-core'; import {MathBackendCPU} from '../backend_cpu'; import {assertNotComplex} from '../cpu_util'; + import {uniqueImpl} from './Unique_impl'; -export function unique(args: {inputs: UniqueInputs, backend: MathBackendCPU}): +export function unique( + args: {inputs: UniqueInputs, attrs: UniqueAttrs, backend: MathBackendCPU}): TensorInfo[] { - const {inputs, backend} = args; + const {inputs, attrs, backend} = args; + const {axis} = attrs; const {x} = inputs; assertNotComplex(x, 'unique'); const values = backend.data.get(x.dataId).values; - const {outputValues, indices} = uniqueImpl(values, x.dtype); + const {outputValues, outputShape, indices} = + uniqueImpl(values, axis, x.shape, x.dtype); return [ - backend.makeTensorInfo(x.shape, x.dtype, outputValues), + backend.makeTensorInfo(outputShape, x.dtype, outputValues), backend.makeTensorInfo([indices.length], 'int32', indices), ]; } diff --git a/tfjs-backend-cpu/src/kernels/Unique_impl.ts b/tfjs-backend-cpu/src/kernels/Unique_impl.ts index a442b8b0097..30edc7cbb6a 100644 --- a/tfjs-backend-cpu/src/kernels/Unique_impl.ts +++ b/tfjs-backend-cpu/src/kernels/Unique_impl.ts @@ -15,35 +15,104 @@ * ============================================================================= */ -import {BackendValues, DataType, TypedArray, util} from '@tensorflow/tfjs-core'; - -export function uniqueImpl(values: BackendValues, dtype: DataType): - {outputValues: BackendValues, indices: BackendValues} { - let xValues: TypedArray|string[] = []; - if (dtype === 'string') { - xValues = (values as Uint8Array[]).map(d => util.decodeString(d)); - } else { - xValues = values as TypedArray; +import {BackendValues, DataType, DataValues, TensorBuffer, TypedArray, util} from '@tensorflow/tfjs-core'; + +export function uniqueImpl( + values: BackendValues, axis: number, shape: number[], dtype: DataType): { + outputValues: BackendValues, + outputShape: number[], + indices: BackendValues +} { + // Normalize and validate axis. + const $axis = util.parseAxisParam(axis, shape)[0]; + + // Calculate the new shape that is suitable for extracting data along the + // given axis. + // + // The rank is 3. + // The size of the 1st dimension is the size of all the axes < the given axis. + // The size of the 2nd dimension is the same as the size of the given axis. + // The size of the 3rd dimension is the size of all the axes > the given axis. + // + // For example, for a 4D tensor with shape=[2, 3, 5, 4] and axis=2, the + // newShape would be: [2*3, 5, 4]. + const newShape = [1, shape[0], 1]; + for (let i = 0; i < $axis; i++) { + newShape[0] *= shape[i]; + } + newShape[1] = shape[$axis]; + for (let i = $axis + 1; i < shape.length; i++) { + newShape[2] *= shape[i]; } - // A map from unique value to its index in outputValues. - const uniqueValues = new Map(); - const outputValues = []; - const indices = new Int32Array(xValues.length); - for (let i = 0; i < xValues.length; i++) { - const value = xValues[i]; - if (uniqueValues.has(value)) { - indices[i] = uniqueValues.get(value); + // A map from unique elements (their string representations) to their values + // in "indices" (below). + const uniqueElements: {[key: string]: number} = {}; + // The indices of each unique element in the original tensor along the given + // axis. It is 1D and has the same size as the given axis. + const indices = new Int32Array(shape[$axis]); + // Create a buffer so we can easily extract value at a given location. + const inputBuffer = new TensorBuffer(newShape, dtype, values as TypedArray); + // The indices along the given axis that have unique elements. This is a + // de-duped version of "indices" above. + const uniqueIndices: number[] = []; + const is1DTensor = newShape[0] === 1 && newShape[2] === 1; + for (let i = 0; i < shape[$axis]; i++) { + // Extract values along the axis. + let element: string; + if (is1DTensor) { + // Fast path for 1D tensor input. + element = values[i].toString(); + } else { + const axisValues = []; + for (let m = 0; m < newShape[0]; m++) { + for (let n = 0; n < newShape[2]; n++) { + axisValues.push(inputBuffer.get(m, i, n)); + } + } + element = axisValues.toString(); + } + + // Dedup and update various indices. + if (uniqueElements[element] !== undefined) { + indices[i] = uniqueElements[element]; } else { - const uniqueIndex = uniqueValues.size; - uniqueValues.set(value, uniqueIndex); + const uniqueIndex = Object.keys(uniqueElements).length; + uniqueElements[element] = uniqueIndex; indices[i] = uniqueIndex; - outputValues.push(values[i]); + uniqueIndices.push(i); } } + // Now we know where each of the unique elements are located along the axis + // (uniqueIndices). Extract them from input buffer and store them in the + // output buffer. + const outputTmpShape = newShape.slice(); + outputTmpShape[1] = Object.keys(uniqueElements).length; + const outputBuffer = new TensorBuffer(outputTmpShape, dtype); + uniqueIndices.forEach((uniqueElementIndex, i) => { + for (let m = 0; m < newShape[0]; m++) { + for (let n = 0; n < newShape[2]; n++) { + outputBuffer.set(inputBuffer.get(m, uniqueElementIndex, n), m, i, n); + } + } + }); + + // The output shape can be calculated from the input shape with the size of + // the given axis replaced by the number of unique elements along that axis. + const outputShape = shape.slice(); + outputShape[$axis] = outputTmpShape[1]; + + // Convert strings back to bytes if input is a string tensor. + let outputValues: BackendValues = outputBuffer.values as BackendValues; + if (dtype === 'string' && util.isString(outputValues[0])) { + outputValues = + (outputValues as DataValues as string[]).map(d => util.encodeString(d)); + } + return { - outputValues: outputValues as BackendValues, + outputValues, + outputShape, indices, }; } diff --git a/tfjs-backend-webgl/src/kernels/Unique.ts b/tfjs-backend-webgl/src/kernels/Unique.ts index 62c08bc5a5d..7420999aa6d 100644 --- a/tfjs-backend-webgl/src/kernels/Unique.ts +++ b/tfjs-backend-webgl/src/kernels/Unique.ts @@ -15,23 +15,29 @@ * ============================================================================= */ -import {KernelConfig, KernelFunc, TensorInfo, Unique, UniqueInputs} from '@tensorflow/tfjs-core'; +import {KernelConfig, KernelFunc, TensorInfo, Unique, UniqueAttrs, UniqueInputs} from '@tensorflow/tfjs-core'; import {MathBackendWebGL} from '../backend_webgl'; import {uniqueImplCPU} from '../kernel_utils/shared'; import {assertNotComplex} from '../webgl_util'; -export function unique(args: {inputs: UniqueInputs, backend: MathBackendWebGL}): +export function unique( + args: + {inputs: UniqueInputs, attrs: UniqueAttrs, backend: MathBackendWebGL}): TensorInfo[] { - const {inputs, backend} = args; + const {inputs, attrs, backend} = args; + const {axis} = attrs; const {x} = inputs; assertNotComplex(x, 'unique'); // For now, always forward calculation to the CPU backend. + console.warn( + 'WARNING: ', 'UI might be locked temparaily as data is being downloaded'); const values = backend.readSync(x.dataId); - const {outputValues, indices} = uniqueImplCPU(values, x.dtype); + const {outputValues, outputShape, indices} = + uniqueImplCPU(values, axis, x.shape, x.dtype); return [ - backend.makeTensorInfo(x.shape, x.dtype, outputValues), + backend.makeTensorInfo(outputShape, x.dtype, outputValues), backend.makeTensorInfo([indices.length], 'int32', indices), ]; } diff --git a/tfjs-converter/metadata/kernel2op.json b/tfjs-converter/metadata/kernel2op.json index 88b359c3639..737a37b53f4 100644 --- a/tfjs-converter/metadata/kernel2op.json +++ b/tfjs-converter/metadata/kernel2op.json @@ -508,6 +508,9 @@ "Unique": [ "unique" ], + "UniqueV2": [ + "unique" + ], "Unpack": [ "unstack" ], diff --git a/tfjs-converter/python/tensorflowjs/op_list/evaluation.json b/tfjs-converter/python/tensorflowjs/op_list/evaluation.json index c0b88c07183..5de11223d6d 100644 --- a/tfjs-converter/python/tensorflowjs/op_list/evaluation.json +++ b/tfjs-converter/python/tensorflowjs/op_list/evaluation.json @@ -32,5 +32,21 @@ "type": "tensor" } ] + }, + { + "tfOpName": "UniqueV2", + "category": "evaluation", + "inputs": [ + { + "start": 0, + "name": "x", + "type": "tensor" + }, + { + "start": 1, + "name": "axis", + "type": "number" + } + ] } ] diff --git a/tfjs-converter/src/operations/executors/evaluation_executor.ts b/tfjs-converter/src/operations/executors/evaluation_executor.ts index 157ad1e1861..5a46646155e 100644 --- a/tfjs-converter/src/operations/executors/evaluation_executor.ts +++ b/tfjs-converter/src/operations/executors/evaluation_executor.ts @@ -42,6 +42,13 @@ export const executeOp: InternalOpExecutor = const result = tfOps.unique(x); return [result.values, result.indices]; } + case 'UniqueV2': { + const x = getParamValue('x', node, tensorMap, context) as Tensor; + const axis = + getParamValue('axis', node, tensorMap, context) as number; + const result = tfOps.unique(x, axis); + return [result.values, result.indices]; + } default: throw TypeError(`Node type ${node.op} is not implemented`); } diff --git a/tfjs-converter/src/operations/executors/evaluation_executor_test.ts b/tfjs-converter/src/operations/executors/evaluation_executor_test.ts index ad7e36d5fab..35edbbb01d3 100644 --- a/tfjs-converter/src/operations/executors/evaluation_executor_test.ts +++ b/tfjs-converter/src/operations/executors/evaluation_executor_test.ts @@ -64,5 +64,18 @@ describe('evaluation', () => { expect(tfOps.unique).toHaveBeenCalledWith(input1[0]); }); }); + + describe('UniqueV2', () => { + it('should get called correctly', () => { + node.op = 'UniqueV2'; + node.inputParams['x'] = createTensorAttr(0); + node.inputParams['axis'] = createNumberAttrFromIndex(1); + spyOn(tfOps, 'unique').and.callThrough(); + const xInput = [tfOps.tensor2d([[1], [2]])]; + const axisInput = [tfOps.scalar(1)]; + executeOp(node, {'input1': xInput, 'input2': axisInput}, context); + expect(tfOps.unique).toHaveBeenCalledWith(xInput[0], 1); + }); + }); }); }); diff --git a/tfjs-converter/src/operations/op_list/evaluation.ts b/tfjs-converter/src/operations/op_list/evaluation.ts index 9d631a94de0..2d09bdb5b0f 100644 --- a/tfjs-converter/src/operations/op_list/evaluation.ts +++ b/tfjs-converter/src/operations/op_list/evaluation.ts @@ -34,4 +34,12 @@ export const json: OpMapper[] = [ {'start': 0, 'name': 'x', 'type': 'tensor'}, ], }, + { + 'tfOpName': 'UniqueV2', + 'category': 'evaluation', + 'inputs': [ + {'start': 0, 'name': 'x', 'type': 'tensor'}, + {'start': 1, 'name': 'axis', 'type': 'number'}, + ], + }, ]; diff --git a/tfjs-core/src/kernel_names.ts b/tfjs-core/src/kernel_names.ts index a4e97e5e627..7584a924576 100644 --- a/tfjs-core/src/kernel_names.ts +++ b/tfjs-core/src/kernel_names.ts @@ -748,6 +748,9 @@ export interface TransposeAttrs { export const Unique = 'Unique'; export type UniqueInputs = Pick; +export interface UniqueAttrs { + axis: number; +} export type UnaryInputs = Pick; diff --git a/tfjs-core/src/ops/unique.ts b/tfjs-core/src/ops/unique.ts index e8bb72849e5..9f3937527dc 100644 --- a/tfjs-core/src/ops/unique.ts +++ b/tfjs-core/src/ops/unique.ts @@ -16,17 +16,17 @@ */ import {ENGINE} from '../engine'; -import {Unique, UniqueInputs} from '../kernel_names'; +import {Unique, UniqueAttrs, UniqueInputs} from '../kernel_names'; +import {NamedAttrMap} from '../kernel_registry'; import {Tensor, Tensor1D} from '../tensor'; import {NamedTensorMap} from '../tensor_types'; import {convertToTensor} from '../tensor_util_env'; import {TensorLike} from '../types'; -import * as util from '../util'; import {op} from './operation'; /** - * Finds the unique elements in a 1-D tensor. + * Finds unique elements along an axis of a tensor. * * It returns a tensor `values` containing all of the unique elements along the * `axis` of the given tensor `x` in the same order that they occur along the @@ -34,35 +34,51 @@ import {op} from './operation'; * `indices` the same size as the number of the elements in `x` along the `axis` * dimension. It contains the index in the unique output `values`. * - * For now, only 1-D tensor is supported, and the `axis` parameter is not used. - * Tensors with higher dimensions will be supported in UniqueV2. - * * ```js - * const a = tf.tensor2d([1, 1, 2, 4, 4, 4, 7, 8, 8]); + * // A 1-D tensor + * const a = tf.tensor1d([1, 1, 2, 4, 4, 4, 7, 8, 8]); * const {values, indices} = tf.unique(a); - * values.print(); // [1, 2, 4, 7, 8, 9] + * values.print(); // [1, 2, 4, 7, 8,] * indices.print(); // [0, 0, 1, 2, 2, 2, 3, 4, 4] + * + * // A 2-D tensor with axis=0 + * // + * // 'a' is: [[1, 0, 0], + * // [1, 0, 0], + * // [2, 0, 0]] + * const a = tf.tensor2d([[1, 0, 0], [1, 0, 0], [2, 0, 0]]); + * const {values, indices} = tf.unique(a, 0) + * values.print(); // [[1, 0, 0], + * // [2, 0, 0]] + * indices.print(); // [0, 0, 1] + * + * // A 2-D tensor with axis=1 + * // + * // 'a' is: [[1, 0, 0], + * // [1, 0, 0], + * // [2, 0, 0]] + * const a = tf.tensor2d([[1, 0, 0], [1, 0, 0], [2, 0, 0]]); + * const {values, indices} = tf.unique(a, 1) + * values.print(); // [[1, 0], + * // [1, 0], + * // [2, 0]] + * indices.print(); // [0, 1, 1] * ``` - * @param x 1-D tensor (int32, string, bool). - * @param axis The axis of the tensor to find the unique elements (not used for - * now). - * @returns [uniqueValues, indices (see above)] + * @param x A tensor (int32, string, bool). + * @param axis The axis of the tensor to find the unique elements. + * @returns [uniqueElements, indices] (see above for details) * * @doc {heading: 'Operations', subheading: 'Evaluation'} */ function unique_( - x: T|TensorLike, axis?: number): {values: Tensor1D, indices: Tensor1D} { + x: T|TensorLike, axis = 0): {values: T, indices: Tensor1D} { // x can be of any dtype, thus null as the last argument. const $x = convertToTensor(x, 'x', 'unique', null); - util.assert( - $x.rank === 1, - () => 'unique() currently only supports 1-D tensor ' + - `(got rank ${$x.rank})`); - const inputs: UniqueInputs = {x: $x}; + const attrs: UniqueAttrs = {axis}; const [values, indices] = ENGINE.runKernel( Unique, inputs as {} as NamedTensorMap, - {} /* attrs */) as [Tensor1D, Tensor1D]; + attrs as {} as NamedAttrMap) as [T, Tensor1D]; return {values, indices}; } diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts index 35f0d27c493..c3a740f9c59 100644 --- a/tfjs-core/src/ops/unique_test.ts +++ b/tfjs-core/src/ops/unique_test.ts @@ -52,9 +52,23 @@ describeWithFlags('unique', ALL_ENVS, () => { expectArraysEqual(await indices.data(), [0, 0, 1]); }); - it('throws for non 1-D tensor', () => { - expect(() => tf.unique([[1, 2], [3, 4]])) - .toThrowError( - /unique\(\) currently only supports 1-D tensor.*got rank 2.*/); + it('2d tensor with axis=0', async () => { + const x = tf.tensor2d([[1, 0, 0], [1, 0, 0], [2, 0, 0]]); + const {values, indices} = tf.unique(x, 0); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[0]]); + expectArraysEqual(await values.data(), [[1, 0, 0], [2, 0, 0]]); + expectArraysEqual(await indices.data(), [0, 0, 1]); + }); + + it('2d tensor with axis=1', async () => { + const x = tf.tensor2d([[1, 0, 0, 1], [1, 0, 0, 1], [2, 0, 0, 2]]); + const {values, indices} = tf.unique(x, 1); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[1]]); + expectArraysEqual(await values.data(), [[1, 0], [1, 0], [2, 0]]); + expectArraysEqual(await indices.data(), [0, 1, 1, 0]); }); }); From bcb46bda31b080dd5c39ee3162de805e8dd4e5b9 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Fri, 2 Oct 2020 16:08:48 -0700 Subject: [PATCH 06/11] update supported ops --- tfjs-converter/docs/supported_ops.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tfjs-converter/docs/supported_ops.md b/tfjs-converter/docs/supported_ops.md index 82c80045e25..0047a19d4ad 100644 --- a/tfjs-converter/docs/supported_ops.md +++ b/tfjs-converter/docs/supported_ops.md @@ -167,6 +167,7 @@ |---|---| |TopKV2|topK| |Unique|unique| +|UniqueV2|unique| |Not mapped|confusionMatrix| |Not mapped|topk| From 79120eb5e12cfe4fb29e5adbb84a7546198a8dbd Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Mon, 5 Oct 2020 15:04:31 -0700 Subject: [PATCH 07/11] address comments --- tfjs-backend-cpu/src/kernels/Unique_impl.ts | 47 +++++++++++++- tfjs-backend-webgl/src/kernels/Unique.ts | 3 +- tfjs-core/src/ops/unique.ts | 3 + tfjs-core/src/ops/unique_test.ts | 72 ++++++++++++++++++++- 4 files changed, 122 insertions(+), 3 deletions(-) diff --git a/tfjs-backend-cpu/src/kernels/Unique_impl.ts b/tfjs-backend-cpu/src/kernels/Unique_impl.ts index 30edc7cbb6a..86e96aef8d9 100644 --- a/tfjs-backend-cpu/src/kernels/Unique_impl.ts +++ b/tfjs-backend-cpu/src/kernels/Unique_impl.ts @@ -36,6 +36,51 @@ export function uniqueImpl( // // For example, for a 4D tensor with shape=[2, 3, 5, 4] and axis=2, the // newShape would be: [2*3, 5, 4]. + // + // Note that this is not the final output shape. This will be the shape for an + // intermediate TensorBuffer (see inputBuffer below) to allow us to extract + // values along the given axis. To demonstrate how it works, consider the + // following example: + // + // Input: a 3D tensor, with shape [1, 2, 3] + // [ + // [ + // [1,2,3], + // [4,5,6] + // ] + // ] + // Axis: 2 (the last axis). + // Along axis 2, we expect to extract 3 tensors: [1,4], [2,5], [3,6]. + // + // For this example, newShape would be: [2, 3, 1], where 2 is calculated from + // 1*2. The re-shaped data would look like: + // + // [ + // [ + // [1], [2], [3] + // ], + // [ + // [4], [5], [6] + // ] + // ] + // + // Then, we can construct a 3-level nested loop by the following dimension + // order to extract the values along the axis (dimension1): + // i: dimension1 // 0,1,2 (newShape[1]) + // m: dimension0 // 0,1 (newShape[0]) + // n: dimension2 // 0 (newShape[2]) + // + // m, i, n + // --------- + // Iteration 0: data at [0, 0, 0] => "1" + // Iteration 1: data at [1, 0, 0] => "4" + // We got [1,4]. + // Iteration 2: data at [0, 1, 0] => "2" + // Iteration 3: data at [1, 1, 0] => "5" + // We got [2,5]. + // Iteration 4: data at [0, 2, 0] => "3" + // Iteration 5: data at [1, 2, 0] => "6" + // We got [3,6]. const newShape = [1, shape[0], 1]; for (let i = 0; i < $axis; i++) { newShape[0] *= shape[i]; @@ -70,7 +115,7 @@ export function uniqueImpl( axisValues.push(inputBuffer.get(m, i, n)); } } - element = axisValues.toString(); + element = axisValues.join(','); } // Dedup and update various indices. diff --git a/tfjs-backend-webgl/src/kernels/Unique.ts b/tfjs-backend-webgl/src/kernels/Unique.ts index 7420999aa6d..4fc78a809ec 100644 --- a/tfjs-backend-webgl/src/kernels/Unique.ts +++ b/tfjs-backend-webgl/src/kernels/Unique.ts @@ -32,7 +32,8 @@ export function unique( // For now, always forward calculation to the CPU backend. console.warn( - 'WARNING: ', 'UI might be locked temparaily as data is being downloaded'); + 'WARNING: ', + 'UI might be locked temporarily as data is being downloaded'); const values = backend.readSync(x.dataId); const {outputValues, outputShape, indices} = uniqueImplCPU(values, axis, x.shape, x.dtype); diff --git a/tfjs-core/src/ops/unique.ts b/tfjs-core/src/ops/unique.ts index 9f3937527dc..7957ff715bd 100644 --- a/tfjs-core/src/ops/unique.ts +++ b/tfjs-core/src/ops/unique.ts @@ -22,6 +22,7 @@ import {Tensor, Tensor1D} from '../tensor'; import {NamedTensorMap} from '../tensor_types'; import {convertToTensor} from '../tensor_util_env'; import {TensorLike} from '../types'; +import {assert} from '../util'; import {op} from './operation'; @@ -74,6 +75,8 @@ function unique_( x: T|TensorLike, axis = 0): {values: T, indices: Tensor1D} { // x can be of any dtype, thus null as the last argument. const $x = convertToTensor(x, 'x', 'unique', null); + assert($x.rank > 0, () => 'The input tensor must be at least 1D'); + const inputs: UniqueInputs = {x: $x}; const attrs: UniqueAttrs = {axis}; const [values, indices] = ENGINE.runKernel( diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts index c3a740f9c59..c31757eff79 100644 --- a/tfjs-core/src/ops/unique_test.ts +++ b/tfjs-core/src/ops/unique_test.ts @@ -28,6 +28,7 @@ describeWithFlags('unique', ALL_ENVS, () => { expect(indices.dtype).toBe('int32'); expect(indices.shape).toEqual(x.shape); + expect(values.shape).toEqual([5]); expectArraysEqual(await values.data(), [1, 2, 4, 7, 8]); expectArraysEqual(await indices.data(), [0, 0, 1, 2, 2, 2, 3, 4, 4]); }); @@ -38,6 +39,8 @@ describeWithFlags('unique', ALL_ENVS, () => { expect(indices.dtype).toBe('int32'); expect(indices.shape).toEqual(x.shape); + expect(values.dtype).toEqual('string'); + expect(values.shape).toEqual([3]); expectArraysEqual(await values.data(), ['a', 'b', 'c']); expectArraysEqual(await indices.data(), [0, 1, 1, 2, 2]); }); @@ -48,6 +51,8 @@ describeWithFlags('unique', ALL_ENVS, () => { expect(indices.dtype).toBe('int32'); expect(indices.shape).toEqual(x.shape); + expect(values.dtype).toEqual('bool'); + expect(values.shape).toEqual([2]); expectArraysEqual(await values.data(), [true, false]); expectArraysEqual(await indices.data(), [0, 0, 1]); }); @@ -58,7 +63,8 @@ describeWithFlags('unique', ALL_ENVS, () => { expect(indices.dtype).toBe('int32'); expect(indices.shape).toEqual([x.shape[0]]); - expectArraysEqual(await values.data(), [[1, 0, 0], [2, 0, 0]]); + expect(values.shape).toEqual([2, 3]); + expectArraysEqual(await values.data(), [1, 0, 0, 2, 0, 0]); expectArraysEqual(await indices.data(), [0, 0, 1]); }); @@ -68,7 +74,71 @@ describeWithFlags('unique', ALL_ENVS, () => { expect(indices.dtype).toBe('int32'); expect(indices.shape).toEqual([x.shape[1]]); + expect(values.shape).toEqual([3, 2]); expectArraysEqual(await values.data(), [[1, 0], [1, 0], [2, 0]]); expectArraysEqual(await indices.data(), [0, 1, 1, 0]); }); + + it('2d tensor with string', async () => { + const x = tf.tensor2d([['a', 'b', 'b'], ['a', 'b', 'b'], ['c', 'b', 'b']]); + const {values, indices} = tf.unique(x, 0); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[0]]); + expect(values.dtype).toEqual('string'); + expect(values.shape).toEqual([2, 3]); + expectArraysEqual(await values.data(), ['a', 'b', 'b', 'c', 'b', 'b']); + expectArraysEqual(await indices.data(), [0, 0, 1]); + }); + + it('3d tensor with axis=0', async () => { + const x = + tf.tensor3d([[[1, 0], [1, 0]], [[1, 0], [1, 0]], [[1, 1], [1, 1]]]); + const {values, indices} = tf.unique(x, 0); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[0]]); + expect(values.shape).toEqual([2, 2, 2]); + expectArraysEqual(await values.data(), [1, 0, 1, 0, 1, 1, 1, 1]); + expectArraysEqual(await indices.data(), [0, 0, 1]); + }); + + it('3d tensor with axis=1', async () => { + const x = + tf.tensor3d([[[1, 0], [1, 0]], [[1, 0], [1, 0]], [[1, 1], [1, 1]]]); + const {values, indices} = tf.unique(x, 1); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[1]]); + expect(values.shape).toEqual([3, 1, 2]); + expectArraysEqual(await values.data(), [[[1, 0]], [[1, 0]], [[1, 1]]]); + expectArraysEqual(await indices.data(), [0, 0]); + }); + + it('3d tensor with axis=2', async () => { + const x = tf.tensor3d([[[1, 0, 1]], [[1, 0, 1]]]); + const {values, indices} = tf.unique(x, 2); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[2]]); + expect(values.shape).toEqual([2, 1, 2]); + expectArraysEqual(await values.data(), [1, 0, 1, 0]); + expectArraysEqual(await indices.data(), [0, 1, 0]); + }); + + it('3d tensor with string', async () => { + const x = tf.tensor3d([ + [['a', 'b'], ['a', 'b']], [['a', 'b'], ['a', 'b']], + [['a', 'a'], ['a', 'a']] + ]); + const {values, indices} = tf.unique(x, 0); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[0]]); + expect(values.dtype).toEqual('string'); + expect(values.shape).toEqual([2, 2, 2]); + expectArraysEqual( + await values.data(), ['a', 'b', 'a', 'b', 'a', 'a', 'a', 'a']); + expectArraysEqual(await indices.data(), [0, 0, 1]); + }); }); From 23d1f41cd90aee6a7c7dd8756105aaf2b335b444 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Tue, 6 Oct 2020 09:16:22 -0700 Subject: [PATCH 08/11] add test and fix js comment --- tfjs-core/src/ops/unique.ts | 4 ++++ tfjs-core/src/ops/unique_test.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tfjs-core/src/ops/unique.ts b/tfjs-core/src/ops/unique.ts index 7957ff715bd..e7d35876a4c 100644 --- a/tfjs-core/src/ops/unique.ts +++ b/tfjs-core/src/ops/unique.ts @@ -41,7 +41,9 @@ import {op} from './operation'; * const {values, indices} = tf.unique(a); * values.print(); // [1, 2, 4, 7, 8,] * indices.print(); // [0, 0, 1, 2, 2, 2, 3, 4, 4] + * ``` * + * ```js * // A 2-D tensor with axis=0 * // * // 'a' is: [[1, 0, 0], @@ -52,7 +54,9 @@ import {op} from './operation'; * values.print(); // [[1, 0, 0], * // [2, 0, 0]] * indices.print(); // [0, 0, 1] + * ``` * + * ```js * // A 2-D tensor with axis=1 * // * // 'a' is: [[1, 0, 0], diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts index c31757eff79..50f4928f3c0 100644 --- a/tfjs-core/src/ops/unique_test.ts +++ b/tfjs-core/src/ops/unique_test.ts @@ -91,6 +91,18 @@ describeWithFlags('unique', ALL_ENVS, () => { expectArraysEqual(await indices.data(), [0, 0, 1]); }); + it('2d tensor with strings that has comma', async () => { + const x = tf.tensor2d([['a', 'b,c', 'd'], ['a', 'b', 'c,d']]); + const {values, indices} = tf.unique(x, 0); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual([x.shape[0]]); + expect(values.dtype).toEqual('string'); + expect(values.shape).toEqual([2, 3]); + expectArraysEqual(await values.data(), ['a', 'b,c', 'd', 'a', 'b', 'c,d']); + expectArraysEqual(await indices.data(), [0, 1]); + }); + it('3d tensor with axis=0', async () => { const x = tf.tensor3d([[[1, 0], [1, 0]], [[1, 0], [1, 0]], [[1, 1], [1, 1]]]); From acb981dd7343d22476797f1a7f5e68af526d4179 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Tue, 6 Oct 2020 09:28:10 -0700 Subject: [PATCH 09/11] remove unnecessary code --- tfjs-backend-cpu/src/kernels/Unique_impl.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tfjs-backend-cpu/src/kernels/Unique_impl.ts b/tfjs-backend-cpu/src/kernels/Unique_impl.ts index 86e96aef8d9..e1e104ca07d 100644 --- a/tfjs-backend-cpu/src/kernels/Unique_impl.ts +++ b/tfjs-backend-cpu/src/kernels/Unique_impl.ts @@ -15,7 +15,7 @@ * ============================================================================= */ -import {BackendValues, DataType, DataValues, TensorBuffer, TypedArray, util} from '@tensorflow/tfjs-core'; +import {BackendValues, DataType, TensorBuffer, TypedArray, util} from '@tensorflow/tfjs-core'; export function uniqueImpl( values: BackendValues, axis: number, shape: number[], dtype: DataType): { @@ -148,15 +148,8 @@ export function uniqueImpl( const outputShape = shape.slice(); outputShape[$axis] = outputTmpShape[1]; - // Convert strings back to bytes if input is a string tensor. - let outputValues: BackendValues = outputBuffer.values as BackendValues; - if (dtype === 'string' && util.isString(outputValues[0])) { - outputValues = - (outputValues as DataValues as string[]).map(d => util.encodeString(d)); - } - return { - outputValues, + outputValues: outputBuffer.values as BackendValues, outputShape, indices, }; From 6360b6509e2708afdc9138793eb1d71dddc80bec Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Tue, 6 Oct 2020 09:30:35 -0700 Subject: [PATCH 10/11] typo --- tfjs-core/src/ops/unique_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts index 50f4928f3c0..a8058f49f23 100644 --- a/tfjs-core/src/ops/unique_test.ts +++ b/tfjs-core/src/ops/unique_test.ts @@ -91,7 +91,7 @@ describeWithFlags('unique', ALL_ENVS, () => { expectArraysEqual(await indices.data(), [0, 0, 1]); }); - it('2d tensor with strings that has comma', async () => { + it('2d tensor with strings that have comma', async () => { const x = tf.tensor2d([['a', 'b,c', 'd'], ['a', 'b', 'c,d']]); const {values, indices} = tf.unique(x, 0); From 130db96745cdc1d157d9fa6139a7cc454aac39f5 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Tue, 6 Oct 2020 15:07:16 -0700 Subject: [PATCH 11/11] add nan/infinity test and disable unique tests in node --- tfjs-core/src/ops/unique_test.ts | 11 +++++++++++ tfjs-node/src/run_tests.ts | 8 ++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tfjs-core/src/ops/unique_test.ts b/tfjs-core/src/ops/unique_test.ts index a8058f49f23..127413b295b 100644 --- a/tfjs-core/src/ops/unique_test.ts +++ b/tfjs-core/src/ops/unique_test.ts @@ -57,6 +57,17 @@ describeWithFlags('unique', ALL_ENVS, () => { expectArraysEqual(await indices.data(), [0, 0, 1]); }); + it('1d tensor with NaN and Infinity', async () => { + const x = tensor1d([NaN, Infinity, NaN, Infinity]); + const {values, indices} = tf.unique(x); + + expect(indices.dtype).toBe('int32'); + expect(indices.shape).toEqual(x.shape); + expect(values.shape).toEqual([2]); + expectArraysEqual(await values.data(), [NaN, Infinity]); + expectArraysEqual(await indices.data(), [0, 1, 0, 1]); + }); + it('2d tensor with axis=0', async () => { const x = tf.tensor2d([[1, 0, 0], [1, 0, 0], [2, 0, 0]]); const {values, indices} = tf.unique(x, 0); diff --git a/tfjs-node/src/run_tests.ts b/tfjs-node/src/run_tests.ts index 6d41a161930..132b84a2851 100644 --- a/tfjs-node/src/run_tests.ts +++ b/tfjs-node/src/run_tests.ts @@ -69,7 +69,8 @@ const IGNORE_LIST: string[] = [ // tslint:disable-next-line:max-line-length 'maxPool3d test-tensorflow {} x=[1,2,2,2,1] f=[2,2,2] s=1 p=1 roundingMode=floor', // libtensorflow doesn't support 6D ArgMax yet. - 'argmax test-tensorflow {} 6D, axis=0', 'diag test-tensorflow {} complex', + 'argmax test-tensorflow {} 6D, axis=0', + 'diag test-tensorflow {} complex', 'diag test-tensorflow {} bool', // See https://github.com/tensorflow/tfjs/issues/1891 'conv2d test-tensorflow {} x=[2,1,2,2] f=[1,1,1,1] s=1 d=1 p=0 NCHW', @@ -79,7 +80,10 @@ const IGNORE_LIST: string[] = [ 'conv2d test-tensorflow {} x=[2,1,2,2] f=[2,2,1,1] s=1 d=1 p=same NCHW', 'conv2d test-tensorflow {} gradient x=[1,1,3,3] f=[2,2,1,1] s=1 p=0 NCHW', 'conv2d test-tensorflow {} gradient x=[2,1,3,3] f=[2,2,1,1] s=1 p=0 NCHW', - 'maxPoolWithArgmax', 'rotate', 'flipLeftRight' + 'maxPoolWithArgmax', + 'rotate', + 'flipLeftRight', + 'unique', ]; if (process.platform === 'win32') {