/
interfaces.ts
296 lines (254 loc) · 9.46 KB
/
interfaces.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
import {
DataTypeFieldConfig, FieldType, DataTypeFields,
ReadonlyDataTypeFields, ReadonlyDataTypeConfig,
} from '@terascope/types';
import { Column } from '../column';
export type { InNumberRangeArg } from '@terascope/utils';
export enum FunctionDefinitionType {
FIELD_TRANSFORM = 'FIELD_TRANSFORM',
RECORD_TRANSFORM = 'RECORD_TRANSFORM',
FIELD_VALIDATION = 'FIELD_VALIDATION',
// eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values
RECORD_VALIDATION = 'RECORD_TRANSFORM'
}
export enum ProcessMode {
/** This indicates that it operations on non-nil individual values */
INDIVIDUAL_VALUES = 'INDIVIDUAL_VALUES',
/** This indicates that it operations on entire values, including nulls and arrays */
FULL_VALUES = 'FULL_VALUES',
/** This indicates a noop, usually this is for metadata/datatype changes */
NONE = 'NONE'
}
export enum FunctionDefinitionCategory {
BOOLEAN = 'BOOLEAN',
GEO = 'GEO',
JSON = 'JSON',
NUMERIC = 'NUMERIC',
OBJECT = 'OBJECT',
STRING = 'STRING',
DATE = 'DATE',
IP = 'IP',
}
export interface FunctionDefinitionExample<T extends Record<string, any>, O = unknown> {
/**
* The example arguments passed to the function
*/
readonly args: T;
/**
* The example data type config and children
*/
readonly config: ReadonlyDataTypeConfig;
/**
* The field to validate against and get the config for.
* Only required for field operations;
*/
readonly field?: string;
/**
* An example input value that will be pretty printed for documentation.
* @note this is only a single value
*/
readonly input: unknown;
/**
* The outputted value that will be pretty printed for documentation.
* In the case of validators, this should be either
* the input or null (which indicates it is invalid)
*/
readonly output?: O;
/**
* Serialize the output for documentation or the function adapter.
* In the functionTestHarness this won't be called out the result
* from the dataFrameAdapter
*/
readonly serialize_output?: (output: O) => unknown;
/**
* If this is set to true, the output is not required. If output
* is specified it should be the error message
*/
readonly fails?: boolean;
/**
* Optionally describe the behavior of this example
*/
readonly description?: string;
/**
* Setting this to true will be exclude it from the
* documentation
*/
readonly test_only?: boolean;
}
export interface FunctionDefinitionConfig<T extends Record<string, any>> {
/**
* The name of the function, this should be considered case-insensitive,
* since some languages like SQL are case insensitive.
*/
readonly name: string;
/**
* Optionally specify other known aliases to this function
*/
readonly aliases?: readonly string[];
/** Type of operation that will be preformed */
readonly type: FunctionDefinitionType;
/** Used to generate documentation */
readonly description: string;
/**
* The category of operation, for documentation purposes
*/
readonly category: FunctionDefinitionCategory;
/**
* Examples that will be used in the documentation and potentially
* in the automated tests.
* FIXME make this non-optional
*/
readonly examples?: readonly FunctionDefinitionExample<T>[];
/**
* Used for validating and defining the types of the input arguments,
* please include description field when creating the schema
*/
readonly argument_schema?: DataTypeFields;
/**
* Used to determine what of the possible args are required, as DataType configs does not have
* a mechanism to specify what is required
*/
readonly required_arguments?: readonly string[];
/**
* Can be used in strongly typed contexts to throw early, some types
* or only compatible with a given operation
*/
readonly accepts: readonly FieldType[];
/** Used for additional custom validation of args, called after generic arg validation */
readonly validate_arguments?: (args: T) => void;
}
export interface FunctionContext<T extends Record<string, any> = Record<string, unknown>> {
readonly args: T,
readonly inputConfig?: DataTypeFieldAndChildren,
readonly outputConfig?: DataTypeFieldAndChildren,
readonly parent: Column<unknown>|unknown[]
}
export interface DynamicFrameFunctionContext<
T extends Record<string, any> = Record<string, unknown>
> {
readonly args: (index: number) => T,
readonly inputConfig?: DataTypeFieldAndChildren,
readonly outputConfig?: DataTypeFieldAndChildren,
readonly parent: Column<unknown>
}
export interface DynamicFunctionContext<T extends Record<string, any> = Record<string, unknown>> {
readonly args: (index: number) => T,
readonly inputConfig?: DataTypeFieldAndChildren,
readonly outputConfig?: DataTypeFieldAndChildren,
readonly parent: unknown[]
}
export interface InitialFunctionContext<T extends Record<string, any> = Record<string, unknown>> {
readonly args: T | ((index: number) => T),
readonly inputConfig?: DataTypeFieldAndChildren,
readonly preserveNulls: boolean,
readonly preserveEmptyObjects: boolean,
readonly field?: string
}
export interface FieldValidateConfig<
T extends Record<string, any> = Record<string, unknown>
> extends FunctionDefinitionConfig<T> {
readonly type: FunctionDefinitionType.FIELD_VALIDATION;
readonly process_mode: ProcessMode;
readonly create: (config: FunctionContext<T>) => (value: unknown, index: number) => boolean;
readonly output_type?: (
inputConfig: DataTypeFieldAndChildren,
args: T
) => DataTypeFieldAndChildren;
}
export interface FieldTransformConfig<
T extends Record<string, any> = Record<string, unknown>
> extends FunctionDefinitionConfig<T> {
readonly type: FunctionDefinitionType.FIELD_TRANSFORM,
readonly process_mode: ProcessMode,
readonly output_type?: (
inputConfig: DataTypeFieldAndChildren,
args: T
) => DataTypeFieldAndChildren;
readonly create: (config: FunctionContext<T>) => (value: unknown, index: number) => unknown;
}
export interface FieldMetaTransform<
T extends Record<string, any> = Record<string, unknown>
> extends FunctionDefinitionConfig<T> {
readonly type: FunctionDefinitionType.FIELD_TRANSFORM;
readonly process_mode: ProcessMode.NONE;
readonly create: (config: FunctionContext<T>) => void;
readonly output_type?: (
inputConfig: DataTypeFieldAndChildren,
args: T
) => DataTypeFieldAndChildren;
}
export interface RecordTransformConfig<
T extends Record<string, any> = Record<string, unknown>
> extends FunctionDefinitionConfig<T> {
readonly type: FunctionDefinitionType.RECORD_TRANSFORM,
readonly output_type: (
inputConfig: ReadonlyDataTypeFields,
args?: T
) => ReadonlyDataTypeFields,
readonly create: (
config: FunctionContext<T>
) => (value: Record<string, unknown>, index: number) => Record<string, unknown>
}
export interface RecordValidationConfig<
T extends Record<string, any> = Record<string, unknown>
> extends FunctionDefinitionConfig<T> {
readonly type: FunctionDefinitionType.RECORD_VALIDATION,
readonly create: (config: FunctionContext<T>) =>
(value: Record<string, unknown>, index: number) => boolean
}
export interface OutputType<T> {
output_type: (
inputConfig: DataTypeFieldAndChildren,
args?: T
) => DataTypeFieldAndChildren
}
export interface DataTypeFieldAndChildren {
readonly field_config: Readonly<DataTypeFieldConfig>,
readonly child_config?: ReadonlyDataTypeFields
}
// TODO: verify this type
export interface FunctionConfigRepository {
readonly [key: string]: FunctionDefinitionConfig<Record<string, unknown>>;
}
export function isFieldValidation<T extends Record<string, any>>(
input: FunctionDefinitionConfig<T>
): input is FieldValidateConfig<T> {
return input && input.type === FunctionDefinitionType.FIELD_VALIDATION;
}
export function isFieldTransform<T extends Record<string, any>>(
input: FunctionDefinitionConfig<T>
): input is FieldTransformConfig<T> {
return input && input.type === FunctionDefinitionType.FIELD_TRANSFORM;
}
export function isFieldOperation<T extends Record<string, any>>(
input: FunctionDefinitionConfig<T>
): input is (FieldValidateConfig<T> | FieldValidateConfig<T>) {
return isFieldValidation(input) || isFieldTransform(input);
}
export function isRecordTransform<T extends Record<string, any>>(
input: FunctionDefinitionConfig<T>
): input is RecordTransformConfig<T> {
return input && input.type === FunctionDefinitionType.RECORD_TRANSFORM;
}
export function isRecordValidation<T extends Record<string, any>>(
input: FunctionDefinitionConfig<T>
): input is RecordValidationConfig<T> {
return input && input.type === FunctionDefinitionType.RECORD_VALIDATION;
}
export function isTransformOperation<T extends Record<string, any>>(
input: FunctionDefinitionConfig<T>
): input is (RecordTransformConfig<T> | FieldTransformConfig<T>) {
return isFieldTransform(input) || isRecordValidation(input);
}
const numericTypes = [
FieldType.Long,
FieldType.Number,
FieldType.Byte,
FieldType.Double,
FieldType.Float,
FieldType.Integer,
FieldType.Short
];
export function isNumericType(fieldConfig: Readonly<DataTypeFieldConfig>): boolean {
return numericTypes.includes(fieldConfig.type as FieldType);
}