diff --git a/frontend/package.json b/frontend/package.json index 08b298260e3..481e720d976 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -38,6 +38,7 @@ "@codingame/monaco-vscode-java-default-extension": "8.0.4", "@codingame/monaco-vscode-python-default-extension": "8.0.4", "@codingame/monaco-vscode-r-default-extension": "8.0.4", + "@lezer/python": "1.1.18", "@ngneat/until-destroy": "8.1.4", "@ngx-formly/core": "6.3.12", "@ngx-formly/ng-zorro-antd": "6.3.12", diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 21928b77039..78fc75d7cbc 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -156,6 +156,7 @@ import { NzTreeModule } from "ng-zorro-antd/tree"; import { NzTreeViewModule } from "ng-zorro-antd/tree-view"; import { NzNoAnimationModule } from "ng-zorro-antd/core/animation"; import { TreeModule } from "@ali-hm/angular-tree-component"; +import { UiUdfParametersComponent } from "./workspace/component/ui-udf-parameters/ui-udf-parameters.component"; import { ResultExportationComponent } from "./workspace/component/result-exportation/result-exportation.component"; import { ReportGenerationService } from "./workspace/service/report-generation/report-generation.service"; import { SearchBarComponent } from "./dashboard/component/user/search-bar/search-bar.component"; @@ -265,6 +266,7 @@ registerLocaleData(en); NzGridModule, ScrollingModule, FormlyRepeatDndComponent, + UiUdfParametersComponent, AdminGmailComponent, PublicProjectComponent, WorkspaceComponent, diff --git a/frontend/src/app/common/formly/formly-config.ts b/frontend/src/app/common/formly/formly-config.ts index c3995abb544..707ddfa7975 100644 --- a/frontend/src/app/common/formly/formly-config.ts +++ b/frontend/src/app/common/formly/formly-config.ts @@ -27,6 +27,7 @@ import { PresetWrapperComponent } from "./preset-wrapper/preset-wrapper.componen import { DatasetFileSelectorComponent } from "../../workspace/component/dataset-file-selector/dataset-file-selector.component"; import { CollabWrapperComponent } from "./collab-wrapper/collab-wrapper/collab-wrapper.component"; import { FormlyRepeatDndComponent } from "./repeat-dnd/repeat-dnd.component"; +import { UiUdfParametersComponent } from "../../workspace/component/ui-udf-parameters/ui-udf-parameters.component"; import { DatasetVersionSelectorComponent } from "../../workspace/component/dataset-version-selector/dataset-version-selector.component"; /** @@ -80,6 +81,7 @@ export const TEXERA_FORMLY_CONFIG = { { name: "inputautocomplete", component: DatasetFileSelectorComponent, wrappers: ["form-field"] }, { name: "datasetversionselector", component: DatasetVersionSelectorComponent, wrappers: ["form-field"] }, { name: "repeat-section-dnd", component: FormlyRepeatDndComponent }, + { name: "ui-udf-parameters", component: UiUdfParametersComponent, wrappers: ["form-field"] }, ], wrappers: [ { name: "preset-wrapper", component: PresetWrapperComponent }, diff --git a/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.html b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.html new file mode 100644 index 00000000000..c56391ab474 --- /dev/null +++ b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.html @@ -0,0 +1,43 @@ + +
+
+
+ {{ column.label }} +
+
+ +
+ +
+ +
+ +
+
diff --git a/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.scss b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.scss new file mode 100644 index 00000000000..ab6b2ad6ee1 --- /dev/null +++ b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.scss @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.ui-udf-parameter-row { + display: grid; + grid-template-columns: 250px 250px 1fr; + gap: 12px; + align-items: start; +} + +.field-cell { + min-width: 0; +} + +:host ::ng-deep .ant-form-item { + margin-bottom: 0; +} + +:host ::ng-deep .ant-form-item-label { + display: none; +} diff --git a/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.spec.ts b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.spec.ts new file mode 100644 index 00000000000..74b876d3218 --- /dev/null +++ b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.spec.ts @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { FormControl } from "@angular/forms"; +import { FormlyFieldConfig } from "@ngx-formly/core"; +import { UiUdfParametersComponent } from "./ui-udf-parameters.component"; + +describe("UiUdfParametersComponent", () => { + let component: UiUdfParametersComponent; + + beforeEach(() => { + component = new UiUdfParametersComponent(); + }); + + it("should disable name and type fields while leaving value editable", () => { + const valueControl = new FormControl({ value: "42", disabled: true }); + const nameControl = new FormControl("threshold"); + const typeControl = new FormControl("double"); + + const rowField = rowConfig([ + { key: "value", formControl: valueControl }, + { key: "attributeName", formControl: nameControl }, + { key: "attributeType", formControl: typeControl }, + ]); + + (component as any).field = { model: [{}], fieldGroup: [rowField] } as FormlyFieldConfig; + + component.onPopulate((component as any).field); + + // templateOptions is deprecated, but some existing Formly wrappers still read it. + [ + { + column: component.fieldColumns[0], + field: component.getColumnField(rowField, component.fieldColumns[0]), + control: valueControl, + }, + { + column: component.fieldColumns[1], + field: component.getColumnField(rowField, component.fieldColumns[1]), + control: nameControl, + }, + { + column: component.fieldColumns[2], + field: component.getColumnField(rowField, component.fieldColumns[2]), + control: typeControl, + }, + ].forEach(({ column, field, control }) => { + expect(component.getColumnField(rowField, column)).toBe(field); + const disabled = column.disabled; + expect((field as FormlyFieldConfig).props?.disabled).toBe(disabled); + expect((field as any).templateOptions?.disabled).toBe(disabled); + expect((control as FormControl).disabled).toBe(disabled); + }); + }); + + it("should apply disabled state to rows generated from the field array template", () => { + const field: FormlyFieldConfig = { + model: [{ value: "42", attribute: { attributeName: "threshold", attributeType: "double" } }], + fieldArray: rowConfig([{ key: "value" }, { key: "attributeName" }, { key: "attributeType" }]), + fieldGroup: [], + }; + + component.onPopulate(field); + + const generatedRow = field.fieldGroup?.[0] as FormlyFieldConfig; + const valueControl = new FormControl({ value: "42", disabled: true }); + const nameControl = new FormControl("threshold"); + const typeControl = new FormControl("double"); + + [ + { column: component.fieldColumns[0], control: valueControl }, + { column: component.fieldColumns[1], control: nameControl }, + { column: component.fieldColumns[2], control: typeControl }, + ].forEach(({ column, control }) => { + const columnField = component.getColumnField(generatedRow, column) as FormlyFieldConfig; + Object.assign(columnField, { formControl: control }); + columnField.hooks?.onInit?.(columnField); + + expect(columnField.props?.disabled).toBe(column.disabled); + expect((columnField as any).templateOptions?.disabled).toBe(column.disabled); + expect(control.disabled).toBe(column.disabled); + }); + }); +}); + +function rowConfig(fields: ReadonlyArray<{ key: string; formControl?: FormControl }>): FormlyFieldConfig { + const [valueField, nameField, typeField] = fields.map(field => ({ + key: field.key, + formControl: field.formControl, + })); + + return { + fieldGroup: [ + valueField, + { + key: "attribute", + fieldGroup: [nameField, typeField], + }, + ], + }; +} diff --git a/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.ts b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.ts new file mode 100644 index 00000000000..d725004c582 --- /dev/null +++ b/frontend/src/app/workspace/component/ui-udf-parameters/ui-udf-parameters.component.ts @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Component } from "@angular/core"; +import { NgFor, NgIf } from "@angular/common"; +import { FieldArrayType, FormlyFieldConfig, FormlyModule } from "@ngx-formly/core"; + +type UiUdfParameterColumn = Readonly<{ label: string; key: string; parentKey?: string; disabled: boolean }>; + +/** Renders inferred Python UDF UI parameters with editable values and locked name/type columns. */ +@Component({ + selector: "texera-ui-udf-parameters", + templateUrl: "./ui-udf-parameters.component.html", + styleUrls: ["./ui-udf-parameters.component.scss"], + imports: [NgIf, NgFor, FormlyModule], +}) +export class UiUdfParametersComponent extends FieldArrayType { + private readonly disabledStateConfigured = new WeakMap(); + + readonly fieldColumns: UiUdfParameterColumn[] = [ + { label: "Value", key: "value", disabled: false }, + { label: "Name", key: "attributeName", parentKey: "attribute", disabled: true }, + { label: "Type", key: "attributeType", parentKey: "attribute", disabled: true }, + ]; + + override onPopulate(field: FormlyFieldConfig): void { + this.configureRowTemplate(this.getFieldArrayTemplate(field)); + super.onPopulate(field); + field.fieldGroup?.forEach(rowField => this.configureRowFields(rowField)); + } + + /** Finds the Formly field config that backs one visible column in a parameter row. */ + getColumnField(rowField: FormlyFieldConfig, column: UiUdfParameterColumn): FormlyFieldConfig | undefined { + return this.getChildField(column.parentKey ? this.getChildField(rowField, column.parentKey) : rowField, column.key); + } + + private getFieldArrayTemplate(field: FormlyFieldConfig): FormlyFieldConfig | undefined { + return typeof field.fieldArray === "function" ? undefined : field.fieldArray; + } + + private configureRowTemplate(rowField: FormlyFieldConfig | undefined): void { + this.configureRowColumns(rowField, this.setDisabledMetadata.bind(this)); + } + + private configureRowFields(rowField: FormlyFieldConfig | undefined): void { + this.configureRowColumns(rowField, this.configureDisabledState.bind(this)); + } + + private configureRowColumns( + rowField: FormlyFieldConfig | undefined, + configureColumn: (field: FormlyFieldConfig | undefined, disabled: boolean) => void + ): void { + if (!rowField) return; + + this.fieldColumns.forEach(column => configureColumn(this.getColumnField(rowField, column), column.disabled)); + } + + private getChildField(rowField: FormlyFieldConfig | undefined, key: string): FormlyFieldConfig | undefined { + return rowField?.fieldGroup?.find(fieldConfig => fieldConfig.key === key); + } + + /** Sets Formly disabled metadata and keeps controls created later in sync through an onInit hook. */ + private configureDisabledState(field: FormlyFieldConfig | undefined, disabled: boolean): void { + if (!field) return; + + this.setDisabledMetadata(field, disabled); + + if (this.disabledStateConfigured.get(field) === disabled) { + this.applyDisabledState(field, disabled); + return; + } + + const previousOnInit = field.hooks?.onInit; + field.hooks = { + ...(field.hooks ?? {}), + onInit: initializedField => { + previousOnInit?.(initializedField); + this.applyDisabledState(initializedField, disabled); + }, + }; + + this.disabledStateConfigured.set(field, disabled); + this.applyDisabledState(field, disabled); + } + + private setDisabledMetadata(field: FormlyFieldConfig | undefined, disabled: boolean): void { + if (!field) return; + + field.props = { ...(field.props ?? {}), disabled }; + + // Keep deprecated templateOptions in sync for existing Formly wrappers that still read it. + (field as any).templateOptions = { ...((field as any).templateOptions ?? {}), disabled }; + } + + private applyDisabledState(field: FormlyFieldConfig, disabled: boolean): void { + if (disabled) field.formControl?.disable({ emitEvent: false }); + else field.formControl?.enable({ emitEvent: false }); + } + + trackByParameterName = (index: number, parameter: any): string | number => { + return parameter?.attribute?.attributeName ?? index; + }; +} diff --git a/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-parser.service.spec.ts b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-parser.service.spec.ts new file mode 100644 index 00000000000..e9fc4e34948 --- /dev/null +++ b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-parser.service.spec.ts @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + UiUdfParametersParseError, + UiUdfParametersParserService, + type UiUdfParameter, +} from "./ui-udf-parameters-parser.service"; + +const MULTIPLE_SUPPORTED_CLASSES_ERROR = "Only one Python UDF class can declare UiParameter values."; +const DUPLICATE_NAME_ERROR = "UiParameter name 'threshold' is declared more than once."; + +describe("UiUdfParametersParserService", () => { + let service: UiUdfParametersParserService; + + beforeEach(() => { + service = new UiUdfParametersParserService(); + }); + + it("should parse supported positional, named, and attr_type arguments", () => { + expectParsed( + service, + ` + self.UiParameter("count", AttributeType.INT) + self.UiParameter(type=AttributeType.STRING, name="name") + self.UiParameter(name="age", type=AttributeType.LONG) + self.UiParameter("score", AttributeType.DOUBLE) + self.UiParameter("enabled", AttributeType.BOOL) + self.UiParameter("created_at", type=AttributeType.TIMESTAMP) + self.UiParameter("alias", attr_type=AttributeType.INTEGER) + `, + [ + parameter("count", "integer"), + parameter("name", "string"), + parameter("age", "long"), + parameter("score", "double"), + parameter("enabled", "boolean"), + parameter("created_at", "timestamp"), + parameter("alias", "integer"), + ] + ); + }); + + it("should parse multiline UiParameter calls with split arguments", () => { + expectParsed( + service, + ` + self.UiParameter( + name= + "threshold", + type= + AttributeType.DOUBLE, + ) + self.UiParameter( + "label", + type= + AttributeType.STRING, + ) + `, + [parameter("threshold", "double"), parameter("label", "string")] + ); + }); + + ( + [ + [ + "ignore calls where name or type is missing", + ` + self.UiParameter(name="a") + self.UiParameter(type=AttributeType.DOUBLE) + `, + [], + ], + [ + "ignore invalid positional argument ordering", + ` + self.UiParameter(AttributeType.INT, "count") + self.UiParameter(name="valid", type=AttributeType.STRING) + `, + [parameter("valid", "string")], + ], + ["ignore legacy key= named argument", 'self.UiParameter(type=AttributeType.DOUBLE, key="a")', []], + [ + "ignore non-self calls and non-AttributeType members", + ` + some.UiParameter("not_self", AttributeType.INT) + self.UiParameter("bad_type", OtherType.INT) + self.UiParameter("valid", AttributeType.STRING) + `, + [parameter("valid", "string")], + ], + [ + "ignore empty and extra positional arguments", + ` + self.UiParameter() + self.UiParameter("too_many", AttributeType.STRING, "extra") + self.UiParameter("valid", AttributeType.STRING) + `, + [parameter("valid", "string")], + ], + [ + "ignore commented out UiParameter calls", + ` + # self.UiParameter("commented", AttributeType.INT) + self.UiParameter("active", AttributeType.INT) # self.UiParameter("trailing", AttributeType.STRING) + `, + [parameter("active", "integer")], + ], + [ + "ignore commented out multiline UiParameter sections", + ` + # self.UiParameter( + # name="commented", + # type=AttributeType.INT, + # ) + self.UiParameter(name="active", type=AttributeType.STRING) + `, + [parameter("active", "string")], + ], + [ + "ignore UiParameter examples inside triple-quoted strings", + ` + """ + self.UiParameter("example", AttributeType.INT) + """ + self.UiParameter("active", AttributeType.DOUBLE) + `, + [parameter("active", "double")], + ], + [ + "reject binary UiParameter types", + ` + self.UiParameter("payload", AttributeType.BINARY) + self.UiParameter("blob", AttributeType.LARGE_BINARY) + `, + [], + ], + ] as ReadonlyArray + ).forEach(([description, openBody, expectedParameters]) => { + it(`should ${description}`, () => { + expectParsed(service, openBody, expectedParameters); + }); + }); + + it("should ignore unsupported classes and custom-named subclasses", () => { + const code = [ + pythonClass('self.UiParameter(type=AttributeType.DOUBLE, name="a")', "RandomClass", "ABC"), + pythonClass('self.UiParameter("threshold", AttributeType.DOUBLE)', "MyTupleOp"), + pythonClass('self.UiParameter("label", AttributeType.STRING)', "MyWrappedTupleOp", "ProcessTupleOperator"), + ].join("\n"); + + expect(service.parse(code)).toEqual([]); + }); + + it("should parse supported UiParameter calls when unsupported classes are present", () => { + const code = [ + pythonClass('self.UiParameter("threshold", AttributeType.DOUBLE)'), + pythonClass('self.UiParameter("ignored", AttributeType.STRING)', "RandomClass", "ABC"), + ].join("\n"); + + expect(service.parse(code)).toEqual([parameter("threshold", "double")]); + }); + + [ + { + description: "multiple supported UDF classes", + code: [ + pythonClass('self.UiParameter("threshold", AttributeType.DOUBLE)', "ProcessTupleOperator"), + pythonClass('self.UiParameter(name="batch_size", type=AttributeType.INT)', "GenerateOperator"), + ].join("\n"), + message: MULTIPLE_SUPPORTED_CLASSES_ERROR, + }, + { + description: "duplicate parameter names", + code: pythonClass(` + self.UiParameter("threshold", AttributeType.DOUBLE) + self.UiParameter("threshold", AttributeType.STRING) + self.UiParameter("label", AttributeType.STRING) + `), + message: DUPLICATE_NAME_ERROR, + }, + ].forEach(({ description, code, message }) => { + it(`should raise an error for ${description}`, () => { + expectParseError(service, code, message); + }); + }); +}); + +function expectParsed( + service: UiUdfParametersParserService, + openBody: string, + expectedParameters: UiUdfParameter[] +): void { + expect(service.parse(pythonClass(openBody))).toEqual(expectedParameters); +} + +function expectParseError(service: UiUdfParametersParserService, code: string, message: string): void { + expect(() => service.parse(code)).toThrow(UiUdfParametersParseError); + expect(() => service.parse(code)).toThrow(message); +} + +function pythonClass(openBody: string, className = "ProcessTupleOperator", baseClass = "UDFOperatorV2"): string { + const openStatements = openBody + .trim() + .split("\n") + .map(line => ` ${line.trim()}`) + .join("\n"); + + return ` + class ${className}(${baseClass}): + def open(self): +${openStatements} + `; +} + +function parameter(attributeName: string, attributeType: UiUdfParameter["attribute"]["attributeType"]): UiUdfParameter { + return { attribute: { attributeName, attributeType }, value: "" }; +} diff --git a/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-parser.service.ts b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-parser.service.ts new file mode 100644 index 00000000000..5c0acb81d09 --- /dev/null +++ b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-parser.service.ts @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Injectable } from "@angular/core"; +import { parser } from "@lezer/python"; +import { AttributeType, SchemaAttribute } from "../../types/workflow-compiling.interface"; + +// Keep in sync with Python UDF template class names in PythonUDFOpDescV2, DualInputPortsPythonUDFOpDescV2, and PythonUDFSourceOpDescV2. +const SUPPORTED_CLASS_NAMES = new Set([ + "ProcessTupleOperator", + "ProcessBatchOperator", + "ProcessTableOperator", + "GenerateOperator", +]); + +const PYTHON_NODE = { + ARG_LIST: "ArgList", + ASSIGN_OP: "AssignOp", + CALL_EXPRESSION: "CallExpression", + CLASS_DEFINITION: "ClassDefinition", + MEMBER_EXPRESSION: "MemberExpression", + PROPERTY_NAME: "PropertyName", + STRING: "String", + VARIABLE_NAME: "VariableName", +} as const; +const ARGUMENT_DELIMITER_NODES = new Set(["(", ")", ","]); + +const UI_PARAMETER_CALLEE = ["self", "UiParameter"]; +const ATTRIBUTE_TYPE_RECEIVER = "AttributeType"; +const ARGUMENT_NAME = "name"; +const ARGUMENT_TYPE = "type"; +const ARGUMENT_ATTR_TYPE = "attr_type"; +const POSITIONAL_ARGUMENT_KEYS = [ARGUMENT_NAME, ARGUMENT_TYPE] as const; + +type ParserSyntaxNode = ReturnType["topNode"]; +type ParsedArgument = Readonly<{ key?: string; value: ParserSyntaxNode }>; +type UiParameterArgument = + | Readonly<{ kind: typeof ARGUMENT_NAME; value: string }> + | Readonly<{ kind: typeof ARGUMENT_TYPE; value: AttributeType }>; + +/** UI parameter row inferred from Python code, with backend-compatible attribute metadata and an editable value. */ +export type UiUdfParameter = Readonly<{ attribute: SchemaAttribute; value: string }>; + +/** Raised when supported Python UDF code declares UI parameters that cannot be represented safely in the UI. */ +export class UiUdfParametersParseError extends Error {} + +// Accept Java enum names (INTEGER, BOOLEAN) and Python enum aliases (INT, BOOL). +const ATTRIBUTE_TYPES_BY_TOKEN: Readonly> = { + STRING: "string", + INTEGER: "integer", + INT: "integer", + LONG: "long", + DOUBLE: "double", + BOOLEAN: "boolean", + BOOL: "boolean", + TIMESTAMP: "timestamp", +}; + +/** Parses Python UDF source code and infers supported self.UiParameter(...) declarations for the property panel. */ +@Injectable({ providedIn: "root" }) +export class UiUdfParametersParserService { + /** + * Returns UI parameters from the single supported Python UDF class in the source. + * Throws UiUdfParametersParseError for duplicate parameter names or multiple supported UDF classes. + */ + parse(code: string): UiUdfParameter[] { + if (!code) return []; + + const result: UiUdfParameter[] = []; + const seen = new Set(); + let supportedClassCount = 0; + let duplicateName: string | undefined; + const addParameter = (parameter?: UiUdfParameter): void => { + const name = parameter?.attribute.attributeName; + if (parameter && name) { + if (seen.has(name)) { + duplicateName = name; + return; + } + seen.add(name); + result.push(parameter); + } + }; + + parser.parse(code).iterate({ + enter: ({ name, node }) => { + const className = node.getChild(PYTHON_NODE.VARIABLE_NAME); + if ( + name !== PYTHON_NODE.CLASS_DEFINITION || + !className || + !SUPPORTED_CLASS_NAMES.has(code.slice(className.from, className.to)) + ) + return; + supportedClassCount++; + node.cursor().iterate(cursorReference => { + if (cursorReference.name !== PYTHON_NODE.CALL_EXPRESSION) return; + addParameter(readCall(cursorReference.node, code)); + return false; + }); + return false; + }, + }); + + if (supportedClassCount > 1) + throw new UiUdfParametersParseError("Only one Python UDF class can declare UiParameter values."); + + if (duplicateName) + throw new UiUdfParametersParseError(`UiParameter name '${duplicateName}' is declared more than once.`); + + return result; + } +} + +function readCall(call: ParserSyntaxNode, code: string): UiUdfParameter | undefined { + const argumentList = call.getChild(PYTHON_NODE.ARG_LIST); + const callee = call.getChild(PYTHON_NODE.MEMBER_EXPRESSION); + if (!argumentList || !isMemberPath(callee, code, UI_PARAMETER_CALLEE)) return undefined; + + let attributeName: string | undefined; + let attributeType: AttributeType | undefined; + const uiParameterArguments = readUiParameterArguments(argumentList, code); + if (!uiParameterArguments) return undefined; + + for (const argument of uiParameterArguments) { + if (argument.kind === ARGUMENT_NAME && !attributeName) attributeName = argument.value; + else if (argument.kind === ARGUMENT_TYPE && !attributeType) attributeType = argument.value; + else return undefined; + } + + return attributeName && attributeType ? { attribute: { attributeName, attributeType }, value: "" } : undefined; +} + +function readUiParameterArguments(argumentList: ParserSyntaxNode, code: string): UiParameterArgument[] | undefined { + const result: UiParameterArgument[] = []; + let positionalIndex = 0; + let sawNamedArgument = false; + + for (const argument of readArguments(argumentList, code)) { + if (argument.key) sawNamedArgument = true; + else if (sawNamedArgument) return undefined; + + const key = argument.key ?? POSITIONAL_ARGUMENT_KEYS[positionalIndex++]; + const parsedArgument = readUiParameterArgument(key, argument.value, code); + if (!parsedArgument) return undefined; + result.push(parsedArgument); + } + + return result; +} + +function readUiParameterArgument( + key: string | undefined, + value: ParserSyntaxNode, + code: string +): UiParameterArgument | undefined { + if (key === ARGUMENT_NAME) { + const attributeName = readName(value, code); + return attributeName ? { kind: ARGUMENT_NAME, value: attributeName } : undefined; + } + if (key === ARGUMENT_TYPE || key === ARGUMENT_ATTR_TYPE) { + const attributeType = readType(value, code); + return attributeType ? { kind: ARGUMENT_TYPE, value: attributeType } : undefined; + } + return undefined; +} + +function readArguments(argumentList: ParserSyntaxNode, code: string): ParsedArgument[] { + const result: ParsedArgument[] = []; + const children = getChildren(argumentList).filter(node => !ARGUMENT_DELIMITER_NODES.has(node.name)); + + for (let index = 0; index < children.length; index++) { + const node = children[index]; + + if (node.name === PYTHON_NODE.VARIABLE_NAME && children[index + 1]?.name === PYTHON_NODE.ASSIGN_OP) { + const value = children[index + 2]; + if (!value) return []; + result.push({ key: code.slice(node.from, node.to), value }); + index += 2; + } else if (node.name !== PYTHON_NODE.ASSIGN_OP) { + result.push({ value: node }); + } else { + return []; + } + } + + return result; +} + +function getChildren(node: ParserSyntaxNode): ParserSyntaxNode[] { + const children: ParserSyntaxNode[] = []; + for (let child = node.firstChild; child; child = child.nextSibling) children.push(child); + return children; +} + +function readName(value: ParserSyntaxNode, code: string): string | undefined { + const name = value.name === PYTHON_NODE.STRING ? readString(code.slice(value.from, value.to))?.trim() : undefined; + return name || undefined; +} + +function readType(value: ParserSyntaxNode, code: string): AttributeType | undefined { + const parts = readMemberPath(value, code); + if (parts?.length !== 2 || parts[0] !== ATTRIBUTE_TYPE_RECEIVER) return undefined; + const token = parts[1].toUpperCase(); + return token ? ATTRIBUTE_TYPES_BY_TOKEN[token] : undefined; +} + +function isMemberPath(node: ParserSyntaxNode | null, code: string, expectedParts: string[]): boolean { + const parts = node ? readMemberPath(node, code) : undefined; + return parts?.length === expectedParts.length && parts.every((part, index) => part === expectedParts[index]); +} + +function readMemberPath(node: ParserSyntaxNode, code: string): string[] | undefined { + if (node.name !== PYTHON_NODE.MEMBER_EXPRESSION) return undefined; + const parts = getChildren(node) + .filter(child => child.name === PYTHON_NODE.VARIABLE_NAME || child.name === PYTHON_NODE.PROPERTY_NAME) + .map(child => code.slice(child.from, child.to)); + return parts.length ? parts : undefined; +} + +function readString(input: string): string | undefined { + return input + .trim() + .match(/^[rRuU]*(?:"""([\s\S]*)"""|'''([\s\S]*)'''|"((?:\\.|[^"\\])*)"|'((?:\\.|[^'\\])*)')$/) + ?.slice(1) + .find(value => value !== undefined); +} diff --git a/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-sync.service.spec.ts b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-sync.service.spec.ts new file mode 100644 index 00000000000..d755978ca6a --- /dev/null +++ b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-sync.service.spec.ts @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; +import { PYTHON_UDF_V2_OP_TYPE } from "../workflow-graph/model/workflow-graph"; +import { UiUdfParametersParseError, UiUdfParametersParserService } from "./ui-udf-parameters-parser.service"; +import type { UiUdfParameter } from "./ui-udf-parameters-parser.service"; +import { UiUdfParametersSyncService } from "./ui-udf-parameters-sync.service"; +import type { Mock } from "vitest"; +import { vi as vitest } from "vitest"; +import * as Yjs from "yjs"; + +describe("UiUdfParametersSyncService", () => { + const operatorId = "operator-1"; + const code = "self.UiParameter(...)"; + + let service: UiUdfParametersSyncService; + let parserServiceMock: { parse: Mock }; + let graphMock: { getOperator: Mock; getSharedOperatorType: Mock }; + let operator: { operatorType: string; operatorProperties: { uiParameters: UiUdfParameter[] } }; + + beforeEach(() => { + operator = { operatorType: PYTHON_UDF_V2_OP_TYPE, operatorProperties: { uiParameters: [] } }; + graphMock = { + getOperator: vitest + .fn() + .mockImplementation((requestedOperatorId: string) => + requestedOperatorId === operatorId ? operator : undefined + ), + getSharedOperatorType: vitest.fn(), + }; + parserServiceMock = { parse: vitest.fn() }; + service = new UiUdfParametersSyncService( + { getTexeraGraph: vitest.fn().mockReturnValue(graphMock) } as unknown as WorkflowActionService, + parserServiceMock as unknown as UiUdfParametersParserService + ); + }); + + [ + { + description: "preserve values from current parameter names", + existingParameters: [parameter("count", "integer", "42")], + parsedParameters: [parameter("count", "integer"), parameter("name", "string")], + expectedParameters: [parameter("count", "integer", "42"), parameter("name", "string", "")], + }, + { + description: "remove stale parameters while preserving retained values", + existingParameters: [parameter("count", "integer", "42"), parameter("removed", "string", "stale")], + parsedParameters: [parameter("count", "integer"), parameter("name", "string")], + expectedParameters: [parameter("count", "integer", "42"), parameter("name", "string", "")], + }, + ].forEach(({ description, existingParameters, parsedParameters, expectedParameters }) => { + it(`should ${description}`, () => { + operator.operatorProperties.uiParameters = existingParameters; + parserServiceMock.parse.mockReturnValue(parsedParameters); + + const parametersChangedObserver = observeParameterChanges(); + + service.syncStructureFromCode(operatorId, code); + + expect(parametersChangedObserver).toHaveBeenCalledWith({ operatorId, parameters: expectedParameters }); + expect(parametersChangedObserver).toHaveBeenCalledOnce(); + }); + }); + + it("should not emit when the merged parameters are unchanged", () => { + operator.operatorProperties.uiParameters = [parameter("count", "integer", "42")]; + parserServiceMock.parse.mockReturnValue([parameter("count", "integer")]); + + const parametersChangedObserver = observeParameterChanges(); + + service.syncStructureFromCode(operatorId, code); + + expect(parametersChangedObserver).not.toHaveBeenCalled(); + }); + + it("should emit parser errors without replacing the current parameters", () => { + operator.operatorProperties.uiParameters = [parameter("count", "integer", "42")]; + parserServiceMock.parse.mockImplementation(() => { + throw new UiUdfParametersParseError("Only one Python UDF class can declare UiParameter values."); + }); + + const parametersChangedObserver = observeParameterChanges(); + const parseErrorObserver = vitest.fn(); + service.uiParametersParseError$.subscribe(parseErrorObserver); + + service.syncStructureFromCode(operatorId, code); + + expect(parametersChangedObserver).not.toHaveBeenCalled(); + expect(parseErrorObserver).toHaveBeenCalledWith({ + operatorId, + message: "Only one Python UDF class can declare UiParameter values.", + }); + }); + + it("should not parse code for non-Python UDF operators", () => { + operator.operatorType = "Projection"; + + const parametersChangedObserver = observeParameterChanges(); + + service.syncStructureFromCode(operatorId, code); + + expect(parserServiceMock.parse).not.toHaveBeenCalled(); + expect(parametersChangedObserver).not.toHaveBeenCalled(); + }); + + it("should read code from the shared operator property when editor code is omitted", () => { + const sharedCode = 'self.UiParameter("count", AttributeType.INT)'; + graphMock.getSharedOperatorType.mockReturnValue(sharedOperatorType(sharedCode)); + parserServiceMock.parse.mockReturnValue([parameter("count", "integer")]); + + const parametersChangedObserver = observeParameterChanges(); + + service.syncStructureFromCode(operatorId); + + expect(parserServiceMock.parse).toHaveBeenCalledWith(sharedCode); + expect(parametersChangedObserver).toHaveBeenCalledWith({ + operatorId, + parameters: [parameter("count", "integer")], + }); + }); + + it("should warn and skip sync when shared code cannot be read", () => { + const sharedCodeError = new Error("missing shared operator"); + const consoleWarnSpy = vitest.spyOn(console, "warn").mockImplementation(() => undefined); + graphMock.getSharedOperatorType.mockImplementation(() => { + throw sharedCodeError; + }); + + try { + service.syncStructureFromCode(operatorId); + + expect(parserServiceMock.parse).not.toHaveBeenCalled(); + expect(consoleWarnSpy).toHaveBeenCalledWith( + "Unable to read Python UDF code from shared operator properties.", + sharedCodeError + ); + } finally { + consoleWarnSpy.mockRestore(); + } + }); + + it("should debounce YText changes and clean up the observer", () => { + vitest.useFakeTimers(); + try { + const sharedCodeText = sharedText('self.UiParameter("count", AttributeType.INT)'); + parserServiceMock.parse.mockReturnValue([parameter("count", "integer")]); + + const parametersChangedObserver = observeParameterChanges(); + const cleanup = service.attachToYCode(operatorId, sharedCodeText); + + expect(parserServiceMock.parse).toHaveBeenCalledOnce(); + expect(parametersChangedObserver).toHaveBeenCalledOnce(); + + sharedCodeText.insert(sharedCodeText.length, "\n# changed"); + vitest.advanceTimersByTime(199); + expect(parserServiceMock.parse).toHaveBeenCalledOnce(); + + vitest.advanceTimersByTime(1); + expect(parserServiceMock.parse).toHaveBeenCalledTimes(2); + expect(parametersChangedObserver).toHaveBeenCalledTimes(2); + + cleanup(); + sharedCodeText.insert(sharedCodeText.length, "\n# after cleanup"); + vitest.advanceTimersByTime(200); + expect(parserServiceMock.parse).toHaveBeenCalledTimes(2); + expect(parametersChangedObserver).toHaveBeenCalledTimes(2); + } finally { + vitest.useRealTimers(); + } + }); + + function observeParameterChanges(): Mock { + const parametersChangedObserver = vitest.fn(); + service.uiParametersChanged$.subscribe(parametersChangedObserver); + return parametersChangedObserver; + } +}); + +function parameter( + attributeName: string, + attributeType: UiUdfParameter["attribute"]["attributeType"], + value = "" +): UiUdfParameter { + return { attribute: { attributeName, attributeType }, value }; +} + +function sharedOperatorType(code: string): Yjs.Map { + const yjsDocument = new Yjs.Doc(); + const sharedOperator = yjsDocument.getMap("operator"); + const operatorProperties = new Yjs.Map(); + + operatorProperties.set("code", sharedText(code)); + sharedOperator.set("operatorProperties", operatorProperties); + return sharedOperator; +} + +function sharedText(text: string): Yjs.Text { + const yjsDocument = new Yjs.Doc(); + const sharedRootMap = yjsDocument.getMap("root"); + const codeText = new Yjs.Text(); + + sharedRootMap.set("code", codeText); + codeText.insert(0, text); + return codeText; +} diff --git a/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-sync.service.ts b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-sync.service.ts new file mode 100644 index 00000000000..519d54aaa97 --- /dev/null +++ b/frontend/src/app/workspace/service/code-editor/ui-udf-parameters-sync.service.ts @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Injectable } from "@angular/core"; +import { isEqual } from "lodash-es"; +import { ReplaySubject, Subject } from "rxjs"; +import { debounceTime } from "rxjs/operators"; +import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; +import { UiUdfParametersParseError, UiUdfParametersParserService } from "./ui-udf-parameters-parser.service"; +import type { UiUdfParameter } from "./ui-udf-parameters-parser.service"; +import { isDefined } from "../../../common/util/predicate"; +import { isPythonUdf } from "../workflow-graph/model/workflow-graph"; +import type { Text as YText } from "yjs"; +import type { YType } from "../../types/shared-editing.interface"; + +type SharedOperatorProperties = Readonly<{ code?: string; [key: string]: unknown }>; + +/** + * Waits briefly after shared-code edits so typing does not parse the full UDF body on every keystroke. + */ +const UI_PARAMETER_SYNC_DEBOUNCE_TIME_MS = 200; + +/** Keeps Python UDF UI parameter structure in sync with the code editor and workflow graph. */ +@Injectable({ providedIn: "root" }) +export class UiUdfParametersSyncService { + private readonly uiParametersChangedSubject = new ReplaySubject<{ operatorId: string; parameters: UiUdfParameter[] }>( + 1 + ); + private readonly uiParametersParseErrorSubject = new ReplaySubject<{ operatorId: string; message?: string }>(1); + + /** Emits when parsed UI parameter structure changes; consumers should write the parameters back to operatorProperties. */ + readonly uiParametersChanged$ = this.uiParametersChangedSubject.asObservable(); + + /** Emits parser errors; an event without message clears the current parse error for that operator. */ + readonly uiParametersParseError$ = this.uiParametersParseErrorSubject.asObservable(); + + constructor( + private workflowActionService: WorkflowActionService, + private uiUdfParametersParserService: UiUdfParametersParserService + ) {} + + /** + * Observes a shared YText code buffer and syncs the initial and debounced future contents. + * Each call attaches an independent observer; call the returned cleanup function to detach it. + */ + attachToYCode(operatorId: string, yCode: YText): () => void { + const codeChanges = new Subject(); + const subscription = codeChanges + .pipe(debounceTime(UI_PARAMETER_SYNC_DEBOUNCE_TIME_MS)) + .subscribe(latestCode => this.syncStructureFromCode(operatorId, latestCode)); + const handler = () => codeChanges.next(yCode.toString()); + + yCode.observe(handler); + this.syncStructureFromCode(operatorId, yCode.toString()); + + return () => { + yCode.unobserve(handler); + subscription.unsubscribe(); + codeChanges.complete(); + }; + } + + /** + * Parses Python UDF code for a known Python UDF operator and emits merged parameter rows when the shape changes. + * If codeFromEditor is omitted, reads from Yjs; does nothing if the operator or code is unavailable. + */ + syncStructureFromCode(operatorId: string, codeFromEditor?: string): void { + const operator = this.workflowActionService.getTexeraGraph().getOperator(operatorId); + + if (!operator || !isPythonUdf(operator)) return; + + const code = codeFromEditor ?? this.getSharedCode(operatorId); + if (!isDefined(code)) return; + + const existingParameters = (operator.operatorProperties?.uiParameters ?? []) as UiUdfParameter[]; + let mergedParameters: UiUdfParameter[]; + + try { + mergedParameters = this.buildParsedShapeWithPreservedValues(code, existingParameters); + } catch (error) { + if (error instanceof UiUdfParametersParseError) { + this.uiParametersParseErrorSubject.next({ operatorId, message: error.message }); + return; + } + throw error; + } + + this.clearParseError(operatorId); + + if (isEqual(existingParameters, mergedParameters)) return; + + this.uiParametersChangedSubject.next({ operatorId, parameters: mergedParameters }); + } + + private buildParsedShapeWithPreservedValues(code: string, existingParameters: UiUdfParameter[]): UiUdfParameter[] { + const parsedParameters = this.uiUdfParametersParserService.parse(code); + const existingValues = new Map( + existingParameters.map(parameter => [parameter.attribute.attributeName, parameter.value] as const) + ); + + return parsedParameters.map(parameter => ({ + ...parameter, + value: existingValues.get(parameter.attribute.attributeName) ?? "", + })); + } + + private getSharedCode(operatorId: string): string | undefined { + try { + const sharedOperatorType = this.workflowActionService.getTexeraGraph().getSharedOperatorType(operatorId); + + const operatorProperties = sharedOperatorType.get("operatorProperties") as YType; + const yCode = operatorProperties.get("code") as unknown as YText; + return yCode?.toString(); + } catch (error) { + console.warn("Unable to read Python UDF code from shared operator properties.", error); + return undefined; + } + } + + private clearParseError(operatorId: string): void { + this.uiParametersParseErrorSubject.next({ operatorId }); + } +} diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 6a4ae4330c4..b929da0c62c 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -3612,6 +3612,42 @@ __metadata: languageName: node linkType: hard +"@lezer/common@npm:^1.0.0, @lezer/common@npm:^1.2.0, @lezer/common@npm:^1.3.0": + version: 1.5.2 + resolution: "@lezer/common@npm:1.5.2" + checksum: 10c0/e39b46d74899409eab549df7942f00cd8c7f46c81ef0e2f079654ca96d262fca009927328bcd500d69270f5f09986e74768bed19c0acaadbd22f1a6c7dd9bd85 + languageName: node + linkType: hard + +"@lezer/highlight@npm:^1.0.0": + version: 1.2.3 + resolution: "@lezer/highlight@npm:1.2.3" + dependencies: + "@lezer/common": "npm:^1.3.0" + checksum: 10c0/3bcb4fce7a1a45b5973895d7cb2be47970a0098700f2a0970aef9878ffd37f540285a2d7388ec1f524726ec90cc5196b5701bbb9764b7e7300786d772b7d2ce2 + languageName: node + linkType: hard + +"@lezer/lr@npm:^1.0.0": + version: 1.4.10 + resolution: "@lezer/lr@npm:1.4.10" + dependencies: + "@lezer/common": "npm:^1.0.0" + checksum: 10c0/15fac0ecc02a57f111432808c89f7cfb9fed0d78d0a98742607acb5859220a9c18526dd6d509d9fe17f5ef762aa73ce29fa6b930739abb857c1df8949a9003ea + languageName: node + linkType: hard + +"@lezer/python@npm:1.1.18": + version: 1.1.18 + resolution: "@lezer/python@npm:1.1.18" + dependencies: + "@lezer/common": "npm:^1.2.0" + "@lezer/highlight": "npm:^1.0.0" + "@lezer/lr": "npm:^1.0.0" + checksum: 10c0/8d984729e887808c75800f18ed54560adfd4e67094b301a1666bdcd49e8987ab45f04c515563a92dfb1377d4a04dcf6616adc50a75285afe9ab53ab90f659bd5 + languageName: node + linkType: hard + "@listr2/prompt-adapter-inquirer@npm:3.0.5": version: 3.0.5 resolution: "@listr2/prompt-adapter-inquirer@npm:3.0.5" @@ -11029,6 +11065,7 @@ __metadata: "@codingame/monaco-vscode-java-default-extension": "npm:8.0.4" "@codingame/monaco-vscode-python-default-extension": "npm:8.0.4" "@codingame/monaco-vscode-r-default-extension": "npm:8.0.4" + "@lezer/python": "npm:1.1.18" "@ngneat/until-destroy": "npm:8.1.4" "@ngx-formly/core": "npm:6.3.12" "@ngx-formly/ng-zorro-antd": "npm:6.3.12"