Bellatrix OpenCV ActionImage Update

bellatrix.android/src/main/java/solutions/bellatrix/android/components/AndroidComponent.java

@@ -22,10 +22,7 @@
 import org.openqa.selenium.interactions.Actions;
 import solutions.bellatrix.android.components.contracts.Component;
 import solutions.bellatrix.android.configuration.AndroidSettings;
-import solutions.bellatrix.android.findstrategies.FindStrategy;
-import solutions.bellatrix.android.findstrategies.NameFindStrategy;
-import solutions.bellatrix.android.findstrategies.TagFindStrategy;
-import solutions.bellatrix.android.findstrategies.XPathFindStrategy;
+import solutions.bellatrix.android.findstrategies.*;
 import solutions.bellatrix.android.infrastructure.DriverService;
 import solutions.bellatrix.android.services.AppService;
 import solutions.bellatrix.android.services.ComponentCreateService;
@@ -36,6 +33,7 @@
 import solutions.bellatrix.core.utilities.DebugInformation;
 import solutions.bellatrix.core.utilities.InstanceFactory;
 import solutions.bellatrix.core.utilities.Log;
+import solutions.bellatrix.plugins.opencv.Base64Encodable;
 
 import java.awt.Dimension;
 import java.util.ArrayList;
@@ -221,6 +219,10 @@ public <TComponent extends AndroidComponent, TFindStrategy extends FindStrategy>
         return createAll(componentClass, findStrategy);
     }
 
+    public <TComponent extends AndroidComponent> TComponent createByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return create(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     public <TComponent extends AndroidComponent> TComponent createByXPath(Class<TComponent> componentClass, String xpath) {
         return create(componentClass, new XPathFindStrategy(xpath));
     }
@@ -237,6 +239,11 @@ public <TComponent extends AndroidComponent> List<TComponent> createAllByName(Cl
         return createAll(componentClass, new NameFindStrategy(name));
     }
 
+    public <TComponent extends AndroidComponent> List<TComponent> createAllByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return createAll(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
+
     public <TComponent extends AndroidComponent> List<TComponent> createAllByXPath(Class<TComponent> componentClass, String xpath) {
         return createAll(componentClass, new XPathFindStrategy(xpath));
     }

bellatrix.android/src/main/java/solutions/bellatrix/android/configuration/AndroidSettings.java

@@ -41,4 +41,6 @@ public class AndroidSettings {
 
     @Getter @Setter private Boolean videosOnFailEnabled;
     @Getter @Setter private String videosSaveLocation;
+
+    @Getter @Setter private Boolean allowImageFindStrategies;
 }

bellatrix.android/src/main/java/solutions/bellatrix/android/findstrategies/ImageBase64FindStrategy.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2025 Automate The Planet Ltd.
+ * Author: Miriyam Kyoseva
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * You may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package solutions.bellatrix.android.findstrategies;
+
+import io.appium.java_client.AppiumBy;
+import io.appium.java_client.android.AndroidDriver;
+import org.openqa.selenium.WebElement;
+import solutions.bellatrix.plugins.opencv.Base64Encodable;
+
+import java.util.List;
+
+public class ImageBase64FindStrategy extends FindStrategy {
+    private final Base64Encodable encodedImage;
+    public ImageBase64FindStrategy(Base64Encodable encodedImage) {
+        super(encodedImage.getImageName());
+        this.encodedImage = encodedImage;
+    }
+
+    @Override
+    public WebElement findElement(AndroidDriver driver) {
+        return driver.findElement(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public List<WebElement> findAllElements(AndroidDriver driver) {
+        return driver.findElements(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public WebElement findElement(WebElement element) {
+        return element.findElement(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public List<WebElement> findAllElements(WebElement element) {
+        return element.findElements(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public String toString() {
+        return String.format("image = %s", getValue());
+    }
+}

bellatrix.android/src/main/java/solutions/bellatrix/android/infrastructure/DriverService.java

@@ -107,6 +107,9 @@ private static AndroidDriver initializeDriverGridMode(GridSettings gridSettings,
         options.put("name", testName);
         caps.setCapability(gridSettings.getOptionsName(), options);
 
+        if (ConfigurationService.get(AndroidSettings.class).getAllowImageFindStrategies())
+            caps.setCapability("use-plugins", "images");
+
         AndroidDriver driver = null;
         try {
             driver = new AndroidDriver(new URL(gridSettings.getUrl()), caps);
@@ -134,6 +137,9 @@ private static AndroidDriver initializeDriverRegularMode(String serviceUrl) {
             caps.setAppActivity(getAppConfiguration().getAppActivity());
         }
 
+        if (ConfigurationService.get(AndroidSettings.class).getAllowImageFindStrategies())
+            caps.setCapability("use-plugins", "images");
+
         addDriverConfigOptions(caps);
         addCustomDriverOptions(caps);
         var driver = new AndroidDriver(new URL(serviceUrl), caps);

bellatrix.android/src/main/java/solutions/bellatrix/android/infrastructure/MobileScreenshotPlugin.java

@@ -13,8 +13,6 @@
 
 package solutions.bellatrix.android.infrastructure;
 
-import lombok.SneakyThrows;
-import org.apache.commons.io.FileUtils;
 import org.openqa.selenium.OutputType;
 import org.openqa.selenium.TakesScreenshot;
 import plugins.screenshots.ScreenshotPlugin;
@@ -35,7 +33,11 @@ public MobileScreenshotPlugin() {
     }
 
     @Override
-    @SneakyThrows
+    public byte[] takeScreenshot() {
+        return ((TakesScreenshot)DriverService.getWrappedAndroidDriver()).getScreenshotAs(OutputType.BYTES);
+    }
+
+    @Override
     public String takeScreenshot(String name) {
         var screenshotSaveDir = getOutputFolder();
         var filename = getUniqueFileName(name);
@@ -57,7 +59,6 @@ public String takeScreenshot(String name) {
     }
 
     @Override
-    @SneakyThrows
     public String takeScreenshot(String screenshotSaveDir, String filename) {
         var screenshot = ((TakesScreenshot)DriverService.getWrappedAndroidDriver()).getScreenshotAs(OutputType.BASE64);
 

bellatrix.android/src/main/java/solutions/bellatrix/android/services/ComponentCreateService.java

@@ -17,6 +17,7 @@
 import solutions.bellatrix.android.findstrategies.*;
 import solutions.bellatrix.android.infrastructure.DriverService;
 import solutions.bellatrix.core.utilities.InstanceFactory;
+import solutions.bellatrix.plugins.opencv.Base64Encodable;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -32,6 +33,10 @@ public <TComponent extends AndroidComponent, TFindStrategy extends FindStrategy>
         return allBy(componentClass, findStrategy);
     }
 
+    public <TComponent extends AndroidComponent> TComponent byImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return by(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     public <TComponent extends AndroidComponent> TComponent byId(Class<TComponent> componentClass, String id) {
         return by(componentClass, new IdFindStrategy(id));
     }
@@ -76,6 +81,10 @@ public <TComponent extends AndroidComponent> TComponent byIdContaining(Class<TCo
         return by(componentClass, new IdContainingFindStrategy(idContaining));
     }
 
+    public <TComponent extends AndroidComponent> List<TComponent> allByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return allBy(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     public <TComponent extends AndroidComponent> List<TComponent> allById(Class<TComponent> componentClass, String automationId) {
         return allBy(componentClass, new IdFindStrategy(automationId));
     }

bellatrix.desktop/pom.xml

@@ -48,5 +48,11 @@
             <version>1.0</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>solutions.bellatrix</groupId>
+            <artifactId>bellatrix.plugins.opencv</artifactId>
+            <version>1.0</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 </project>

bellatrix.desktop/src/main/java/solutions/bellatrix/desktop/components/DesktopComponent.java

@@ -34,6 +34,7 @@
 import solutions.bellatrix.desktop.services.ComponentCreateService;
 import solutions.bellatrix.desktop.services.ComponentWaitService;
 import solutions.bellatrix.desktop.waitstrategies.*;
+import solutions.bellatrix.plugins.opencv.Base64Encodable;
 
 import java.awt.Dimension;
 import java.util.ArrayList;
@@ -247,6 +248,11 @@ public <TComponent extends DesktopComponent> TComponent createByAutomationId(Cla
         return create(componentClass, new AutomationIdFindStrategy(automationId));
     }
 
+    public <TComponent extends DesktopComponent> TComponent createByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return create(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
+
     public <TComponent extends DesktopComponent> List<TComponent> createAllByAccessibilityId(Class<TComponent> componentClass, String accessibilityId) {
         return createAll(componentClass, new AccessibilityIdFindStrategy(accessibilityId));
     }
@@ -275,6 +281,10 @@ public <TComponent extends DesktopComponent> List<TComponent> createAllByIdConta
         return createAll(componentClass, new IdContainingFindStrategy(idContaining));
     }
 
+    public <TComponent extends DesktopComponent> List<TComponent> createAllByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return createAll(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     protected <TComponent extends DesktopComponent, TFindStrategy extends FindStrategy> TComponent create(Class<TComponent> componentClass, TFindStrategy findStrategy) {
         CREATING_ELEMENT.broadcast(new ComponentActionEventArgs(this));
         findElement();

bellatrix.desktop/src/main/java/solutions/bellatrix/desktop/configuration/DesktopSettings.java

@@ -47,4 +47,6 @@ public class DesktopSettings {
 
     @Getter @Setter private Boolean videosOnFailEnabled;
     @Getter @Setter private String videosSaveLocation;
+
+    @Getter @Setter private Boolean allowImageFindStrategies;
 }

bellatrix.desktop/src/main/java/solutions/bellatrix/desktop/findstrategies/ImageBase64FindStrategy.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2025 Automate The Planet Ltd.
+ * Author: Miriyam Kyoseva
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * You may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package solutions.bellatrix.desktop.findstrategies;
+
+import io.appium.java_client.AppiumBy;
+import io.appium.java_client.windows.WindowsDriver;
+import org.openqa.selenium.WebElement;
+import solutions.bellatrix.plugins.opencv.Base64Encodable;
+
+import java.util.List;
+
+public class ImageBase64FindStrategy extends FindStrategy {
+    private final Base64Encodable encodedImage;
+    public ImageBase64FindStrategy(Base64Encodable encodedImage) {
+        super(encodedImage.getImageName());
+        this.encodedImage = encodedImage;
+    }
+
+    @Override
+    public WebElement findElement(WindowsDriver driver) {
+        return driver.findElement(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public List<WebElement> findAllElements(WindowsDriver driver) {
+        return driver.findElements(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public WebElement findElement(WebElement element) {
+        return element.findElement(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public List<WebElement> findAllElements(WebElement element) {
+        return element.findElements(AppiumBy.image(encodedImage.getBase64Image()));
+    }
+
+    @Override
+    public String toString() {
+        return String.format("image = %s", getValue());
+    }
+}

bellatrix.desktop/src/main/java/solutions/bellatrix/desktop/infrastructure/DesktopScreenshotPlugin.java

@@ -13,8 +13,6 @@
 
 package solutions.bellatrix.desktop.infrastructure;
 
-import lombok.SneakyThrows;
-import org.apache.commons.io.FileUtils;
 import org.openqa.selenium.OutputType;
 import org.openqa.selenium.TakesScreenshot;
 import plugins.screenshots.ScreenshotPlugin;
@@ -35,7 +33,11 @@ public DesktopScreenshotPlugin() {
     }
 
     @Override
-    @SneakyThrows
+    public byte[] takeScreenshot() {
+        return ((TakesScreenshot)DriverService.getWrappedDriver()).getScreenshotAs(OutputType.BYTES);
+    }
+
+    @Override
     public String takeScreenshot(String name) {
         var screenshotSaveDir = getOutputFolder();
         var filename = getUniqueFileName(name);
@@ -56,7 +58,6 @@ public String takeScreenshot(String name) {
     }
 
     @Override
-    @SneakyThrows
     public String takeScreenshot(String screenshotSaveDir, String filename) {
         var screenshot = ((TakesScreenshot)DriverService.getWrappedDriver()).getScreenshotAs(OutputType.BASE64);
         Path path = Paths.get(screenshotSaveDir, filename);

bellatrix.desktop/src/main/java/solutions/bellatrix/desktop/infrastructure/DriverService.java

@@ -88,6 +88,9 @@ private static WindowsDriver initializeDriverGridMode(GridSettings gridSettings)
         caps.setApp(getAppConfiguration().getAppPath().replace("\\", "/"));
         caps.setAppWorkingDir(new File(getAppConfiguration().getAppPath()).getParent());
 
+        if (ConfigurationService.get(DesktopSettings.class).getAllowImageFindStrategies())
+            caps.setCapability("use-plugins", "images");
+
         WindowsDriver driver = null;
         try {
             driver = new WindowsDriver(new URL(gridSettings.getUrl()), caps);

bellatrix.desktop/src/main/java/solutions/bellatrix/desktop/services/ComponentCreateService.java

@@ -17,6 +17,7 @@
 import solutions.bellatrix.desktop.components.DesktopComponent;
 import solutions.bellatrix.desktop.findstrategies.*;
 import solutions.bellatrix.desktop.infrastructure.DriverService;
+import solutions.bellatrix.plugins.opencv.Base64Encodable;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -60,6 +61,10 @@ public <TComponent extends DesktopComponent> TComponent byIdContaining(Class<TCo
         return by(componentClass, new IdContainingFindStrategy(idContaining));
     }
 
+    public <TComponent extends DesktopComponent> TComponent byImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return by(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     public <TComponent extends DesktopComponent> List<TComponent> allByAutomationId(Class<TComponent> componentClass, String automationId) {
         return allBy(componentClass, new AutomationIdFindStrategy(automationId));
     }
@@ -88,6 +93,10 @@ public <TComponent extends DesktopComponent> List<TComponent> allByIdContaining(
         return allBy(componentClass, new IdContainingFindStrategy(idContaining));
     }
 
+    public <TComponent extends DesktopComponent> List<TComponent> allByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return allBy(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     public <TComponent extends DesktopComponent, TFindStrategy extends FindStrategy> TComponent by(Class<TComponent> componentClass, TFindStrategy findStrategy) {
         var component = InstanceFactory.create(componentClass);
         component.setFindStrategy(findStrategy);

bellatrix.ios/src/main/java/solutions/bellatrix/ios/components/IOSComponent.java

@@ -33,6 +33,7 @@
 import solutions.bellatrix.ios.services.ComponentCreateService;
 import solutions.bellatrix.ios.services.ComponentWaitService;
 import solutions.bellatrix.ios.waitstrategies.*;
+import solutions.bellatrix.plugins.opencv.Base64Encodable;
 
 import java.awt.Dimension;
 import java.util.ArrayList;
@@ -249,6 +250,10 @@ public <TComponent extends IOSComponent> TComponent createByIOSClassChain(Class<
         return create(componentClass, new IOSClassChainFindStrategy(iosClassChain));
     }
 
+    public <TComponent extends IOSComponent> TComponent createByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return create(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     public <TComponent extends IOSComponent> List<TComponent> createAllByName(Class<TComponent> componentClass, String name) {
         return createAll(componentClass, new NameFindStrategy(name));
     }
@@ -281,6 +286,10 @@ public <TComponent extends IOSComponent> List<TComponent> createAllByIOSClassCha
         return createAll(componentClass, new IOSClassChainFindStrategy(iosClassChain));
     }
 
+    public <TComponent extends IOSComponent> List<TComponent> createAllByImage(Class<TComponent> componentClass, Base64Encodable encodedImage) {
+        return createAll(componentClass, new ImageBase64FindStrategy(encodedImage));
+    }
+
     protected <TComponent extends IOSComponent, TFindStrategy extends FindStrategy> TComponent create(Class<TComponent> componentClass, TFindStrategy findStrategy) {
         CREATING_ELEMENT.broadcast(new ComponentActionEventArgs(this));
         findElement();

bellatrix.ios/src/main/java/solutions/bellatrix/ios/configuration/IOSSettings.java

@@ -50,4 +50,6 @@ public class IOSSettings {
 
     @Getter @Setter private Boolean videosOnFailEnabled;
     @Getter @Setter private String videosSaveLocation;
+
+    @Getter @Setter private Boolean allowImageFindStrategies;
 }