-
Notifications
You must be signed in to change notification settings - Fork 2.7k
/
tensor-factory.ts
320 lines (284 loc) · 11.6 KB
/
tensor-factory.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import {Tensor, TypedTensor} from './tensor.js';
export type ImageFormat = 'RGB'|'RGBA'|'BGR'|'RBG';
export type ImageTensorLayout = 'NHWC'|'NCHW';
// the following region contains type definitions for constructing tensor from a specific location.
// #region types for constructing a tensor from a specific location
/**
* represent common properties of the parameter for constructing a tensor from a specific location.
*/
interface CommonConstructorParameters<T> extends Pick<Tensor, 'dims'> {
/**
* Specify the data type of the tensor.
*/
readonly type: T;
}
/**
* represent the parameter for constructing a tensor from a GPU resource.
*/
interface GpuResourceConstructorParameters<T extends Tensor.Type> {
/**
* an optional callback function to download data from GPU to CPU.
*
* If not provided, the tensor treat the GPU data as external resource.
*/
download?(): Promise<Tensor.DataTypeMap[T]>;
/**
* an optional callback function that will be called when the tensor is disposed.
*
* If not provided, the tensor treat the GPU data as external resource.
*/
dispose?(): void;
}
/**
* represent the parameter for constructing a tensor from a pinned CPU buffer
*/
export interface CpuPinnedConstructorParameters<T extends Tensor.CpuPinnedDataTypes = Tensor.CpuPinnedDataTypes> extends
CommonConstructorParameters<T> {
/**
* Specify the location of the data to be 'cpu-pinned'.
*/
readonly location: 'cpu-pinned';
/**
* Specify the CPU pinned buffer that holds the tensor data.
*/
readonly data: Tensor.DataTypeMap[T];
}
/**
* represent the parameter for constructing a tensor from a WebGL texture
*/
export interface TextureConstructorParameters<T extends Tensor.TextureDataTypes = Tensor.TextureDataTypes> extends
CommonConstructorParameters<T>, GpuResourceConstructorParameters<T> {
/**
* Specify the location of the data to be 'texture'.
*/
readonly location: 'texture';
/**
* Specify the WebGL texture that holds the tensor data.
*/
readonly texture: Tensor.TextureType;
}
/**
* represent the parameter for constructing a tensor from a WebGPU buffer
*/
export interface GpuBufferConstructorParameters<T extends Tensor.GpuBufferDataTypes = Tensor.GpuBufferDataTypes> extends
CommonConstructorParameters<T>, GpuResourceConstructorParameters<T> {
/**
* Specify the location of the data to be 'gpu-buffer'.
*/
readonly location: 'gpu-buffer';
/**
* Specify the WebGPU buffer that holds the tensor data.
*/
readonly gpuBuffer: Tensor.GpuBufferType;
}
// #endregion
// the following region contains type definitions of each individual options.
// the tensor factory functions use a composition of those options as the parameter type.
// #region Options fields
export interface OptionsFormat {
/**
* Describes the image format represented in RGBA color space.
*/
format?: ImageFormat;
}
export interface OptionsTensorFormat {
/**
* Describes the image format of the tensor.
*
* NOTE: this is different from option 'format'. While option 'format' represents the original image, 'tensorFormat'
* represents the target format of the tensor. A transpose will be performed if they are different.
*/
tensorFormat?: ImageFormat;
}
export interface OptionsTensorDataType {
/**
* Describes the data type of the tensor.
*/
dataType?: 'float32'|'uint8';
}
export interface OptionsTensorLayout {
/**
* Describes the tensor layout when representing data of one or more image(s).
*/
tensorLayout?: ImageTensorLayout;
}
export interface OptionsDimensions {
/**
* Describes the image height in pixel
*/
height?: number;
/**
* Describes the image width in pixel
*/
width?: number;
}
export interface OptionResizedDimensions {
/**
* Describes the resized height. If omitted, original height will be used.
*/
resizedHeight?: number;
/**
* Describes resized width - can be accessed via tensor dimensions as well
*/
resizedWidth?: number;
}
export interface OptionsNormalizationParameters {
/**
* Describes normalization parameters when preprocessing the image as model input.
*
* Data element are ranged from 0 to 255.
*/
norm?: {
/**
* The 'bias' value for image normalization.
* - If omitted, use default value 0.
* - If it's a single number, apply to each channel
* - If it's an array of 3 or 4 numbers, apply element-wise. Number of elements need to match the number of channels
* for the corresponding image format
*/
bias?: number|[number, number, number]|[number, number, number, number];
/**
* The 'mean' value for image normalization.
* - If omitted, use default value 255.
* - If it's a single number, apply to each channel
* - If it's an array of 3 or 4 numbers, apply element-wise. Number of elements need to match the number of channels
* for the corresponding image format
*/
mean?: number | [number, number, number] | [number, number, number, number];
};
}
// #endregion
// #region Options composition
export interface TensorFromImageDataOptions extends OptionResizedDimensions, OptionsTensorFormat, OptionsTensorLayout,
OptionsTensorDataType, OptionsNormalizationParameters {}
export interface TensorFromImageElementOptions extends OptionResizedDimensions, OptionsTensorFormat,
OptionsTensorLayout, OptionsTensorDataType,
OptionsNormalizationParameters {}
export interface TensorFromUrlOptions extends OptionsDimensions, OptionResizedDimensions, OptionsTensorFormat,
OptionsTensorLayout, OptionsTensorDataType,
OptionsNormalizationParameters {}
export interface TensorFromImageBitmapOptions extends OptionResizedDimensions, OptionsTensorFormat, OptionsTensorLayout,
OptionsTensorDataType, OptionsNormalizationParameters {}
export interface TensorFromTextureOptions<T extends Tensor.TextureDataTypes> extends
Required<OptionsDimensions>, OptionsFormat, GpuResourceConstructorParameters<T>/* TODO: add more */ {}
export interface TensorFromGpuBufferOptions<T extends Tensor.GpuBufferDataTypes> extends
Pick<Tensor, 'dims'>, GpuResourceConstructorParameters<T> {
/**
* Describes the data type of the tensor.
*/
dataType?: T;
}
// #endregion
/**
* type TensorFactory defines the factory functions of 'Tensor' to create tensor instances from existing data or
* resources.
*/
export interface TensorFactory {
/**
* create a tensor from an ImageData object
*
* @param imageData - the ImageData object to create tensor from
* @param options - An optional object representing options for creating tensor from ImageData.
*
* The following default settings will be applied:
* - `tensorFormat`: `'RGB'`
* - `tensorLayout`: `'NCHW'`
* - `dataType`: `'float32'`
* @returns A promise that resolves to a tensor object
*/
fromImage(imageData: ImageData, options?: TensorFromImageDataOptions):
Promise<TypedTensor<'float32'>|TypedTensor<'uint8'>>;
/**
* create a tensor from a HTMLImageElement object
*
* @param imageElement - the HTMLImageElement object to create tensor from
* @param options - An optional object representing options for creating tensor from HTMLImageElement.
*
* The following default settings will be applied:
* - `tensorFormat`: `'RGB'`
* - `tensorLayout`: `'NCHW'`
* - `dataType`: `'float32'`
* @returns A promise that resolves to a tensor object
*/
fromImage(imageElement: HTMLImageElement, options?: TensorFromImageElementOptions):
Promise<TypedTensor<'float32'>|TypedTensor<'uint8'>>;
/**
* create a tensor from URL
*
* @param urlSource - a string as a URL to the image or a data URL containing the image data.
* @param options - An optional object representing options for creating tensor from URL.
*
* The following default settings will be applied:
* - `tensorFormat`: `'RGB'`
* - `tensorLayout`: `'NCHW'`
* - `dataType`: `'float32'`
* @returns A promise that resolves to a tensor object
*/
fromImage(urlSource: string, options?: TensorFromUrlOptions): Promise<TypedTensor<'float32'>|TypedTensor<'uint8'>>;
/**
* create a tensor from an ImageBitmap object
*
* @param bitmap - the ImageBitmap object to create tensor from
* @param options - An optional object representing options for creating tensor from URL.
*
* The following default settings will be applied:
* - `tensorFormat`: `'RGB'`
* - `tensorLayout`: `'NCHW'`
* - `dataType`: `'float32'`
* @returns A promise that resolves to a tensor object
*/
fromImage(bitmap: ImageBitmap, options: TensorFromImageBitmapOptions):
Promise<TypedTensor<'float32'>|TypedTensor<'uint8'>>;
/**
* create a tensor from a WebGL texture
*
* @param texture - the WebGLTexture object to create tensor from
* @param options - An optional object representing options for creating tensor from WebGL texture.
*
* The options include following properties:
* - `width`: the width of the texture. Required.
* - `height`: the height of the texture. Required.
* - `format`: the format of the texture. If omitted, assume 'RGBA'.
* - `download`: an optional function to download the tensor data from GPU to CPU. If omitted, the GPU data
* will not be able to download. Usually, this is provided by a GPU backend for the inference outputs. Users don't
* need to provide this function.
* - `dispose`: an optional function to dispose the tensor data on GPU. If omitted, the GPU data will not be disposed.
* Usually, this is provided by a GPU backend for the inference outputs. Users don't need to provide this function.
*
* @returns a tensor object
*/
fromTexture<T extends Tensor.TextureDataTypes = 'float32'>(
texture: Tensor.TextureType, options: TensorFromTextureOptions<T>): TypedTensor<'float32'>;
/**
* create a tensor from a WebGPU buffer
*
* @param buffer - the GPUBuffer object to create tensor from
* @param options - An optional object representing options for creating tensor from WebGPU buffer.
*
* The options include following properties:
* - `dataType`: the data type of the tensor. If omitted, assume 'float32'.
* - `dims`: the dimension of the tensor. Required.
* - `download`: an optional function to download the tensor data from GPU to CPU. If omitted, the GPU data
* will not be able to download. Usually, this is provided by a GPU backend for the inference outputs. Users don't
* need to provide this function.
* - `dispose`: an optional function to dispose the tensor data on GPU. If omitted, the GPU data will not be disposed.
* Usually, this is provided by a GPU backend for the inference outputs. Users don't need to provide this function.
*
* @returns a tensor object
*/
fromGpuBuffer<T extends Tensor.GpuBufferDataTypes>(
buffer: Tensor.GpuBufferType, options: TensorFromGpuBufferOptions<T>): TypedTensor<T>;
/**
* create a tensor from a pre-allocated buffer. The buffer will be used as a pinned buffer.
*
* @param type - the tensor element type.
* @param buffer - a TypedArray corresponding to the type.
* @param dims - specify the dimension of the tensor. If omitted, a 1-D tensor is assumed.
*
* @returns a tensor object
*/
fromPinnedBuffer<T extends Exclude<Tensor.Type, 'string'>>(
type: T, buffer: Tensor.DataTypeMap[T], dims?: readonly number[]): TypedTensor<T>;
}