feat(askui-nodejs): poc for extending get command to receive image

askui · Jun 18, 2024 · d2c7421 · d2c7421
1 parent f7adfa0
commit d2c7421
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/packages/askui-nodejs/src/execution/dsl.ts b/packages/askui-nodejs/src/execution/dsl.ts
@@ -71,7 +71,12 @@ abstract class FluentBase {
     if (this instanceof Getter) {
       const getter = this as Getter;
       const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') as CustomElementJson[] : [];
-      return getter.getterExecutor(
+      const imagePath = newParamsList.get('imagePath') as string[] ?? [];
+      return imagePath !== undefined ? getter.getterExecutor(
+        newCurrentInstruction.trim(),
+        customElements,
+        imagePath[0],
+      ) : getter.getterExecutor(
         newCurrentInstruction.trim(),
         customElements,
       );
@@ -3711,11 +3716,12 @@ export abstract class Getter extends FluentCommand {
    *
    * @return {FluentFiltersGetter}
    */
-  get(): FluentFiltersGetter {
+  get(imagePath?: string): FluentFiltersGetter {
     this._textStr = '';
 
     this._textStr += 'get';
     this._textStr += ' element';
+    this._params.set('imagePath', imagePath);
 
     return new FluentFiltersGetter(this);
   }
@@ -3772,6 +3778,7 @@ export abstract class Getter extends FluentCommand {
   abstract getterExecutor(
     instruction: string,
     customElements: CustomElementJson[],
+    imagePath?: string | undefined,
   ): Promise<DetectedElement[]>;
 }
 

diff --git a/packages/askui-nodejs/src/execution/execution-runtime.ts b/packages/askui-nodejs/src/execution/execution-runtime.ts
@@ -184,10 +184,11 @@ export class ExecutionRuntime {
 
   async getDetectedElements(
     instruction: string,
-    customElementJson?: CustomElementJson[],
+    customElementJson?: CustomElementJson[] | undefined,
+    imagePath?: string | undefined,
   ): Promise<DetectedElement[]> {
     let customElements: CustomElement[] = [];
-    const base64Image = await this.takeScreenshotIfImageisNotProvided();
+    const base64Image = await this.takeScreenshotIfImageisNotProvided(imagePath);
     if (customElementJson !== undefined) {
       customElements = await CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
     }

diff --git a/packages/askui-nodejs/src/execution/ui-control-client.ts b/packages/askui-nodejs/src/execution/ui-control-client.ts
@@ -161,11 +161,12 @@ export class UiControlClient extends ApiCommands {
   async getterExecutor(
     instruction: string,
     customElementJson: CustomElementJson[] = [],
+    imagePath?: string | undefined,
   ): Promise<DetectedElement[]> {
     const customElements = await CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
     const stringWithoutSeparators = this.escapeSeparatorString(instruction);
     logger.debug(stringWithoutSeparators);
-    return this.executionRuntime.getDetectedElements(instruction, customElements);
+    return this.executionRuntime.getDetectedElements(instruction, customElements, imagePath);
   }
 
   private secretText: string | undefined = undefined;