Skip to content

Commit

Permalink
[WIP] Rework elements location strategy:
Browse files Browse the repository at this point in the history
- Finding the closest element to matching point instead of the topmost element/ all elements on point
- Support finding multiple elements (multiple image matches)
- Support relative search (e.g. from element)
- Add javadocs
  • Loading branch information
mialeska committed Apr 25, 2023
1 parent 3dcf029 commit 4e82235
Showing 1 changed file with 71 additions and 22 deletions.
93 changes: 71 additions & 22 deletions src/main/java/aquality/selenium/elements/interfaces/ByImage.java
Original file line number Diff line number Diff line change
@@ -1,18 +1,24 @@
package aquality.selenium.elements.interfaces;

import aquality.selenium.browser.AqualityServices;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfByte;
import org.opencv.core.Point;
import org.opencv.core.*;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import org.openqa.selenium.*;
import org.openqa.selenium.interactions.Locatable;

import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;

/**
* Locator to search elements by image.
* Takes screenshot and finds match using openCV.
* Then finds elements by coordinates using javascript.
*/
public class ByImage extends By {
private static boolean wasLibraryLoaded = false;
private final Mat template;
Expand All @@ -25,16 +31,31 @@ private static void loadLibrary() {
}
}

/**
* Constructor accepting image file.
*
* @param file image file to locate element by.
*/
public ByImage(File file) {
loadLibrary();
this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED);
}

/**
* Constructor accepting image file.
*
* @param bytes image bytes to locate element by.
*/
public ByImage(byte[] bytes) {
loadLibrary();
this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED);
}

@Override
public String toString() {
return "ByImage: " + new Dimension(template.width(), template.height());
}

@Override
public List<WebElement> findElements(SearchContext context) {
byte[] screenshotBytes = getScreenshot(context);
Expand All @@ -45,33 +66,61 @@ public List<WebElement> findElements(SearchContext context) {
float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold();
Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result);

if (minMaxLoc.maxVal < threshold) {
AqualityServices.getLogger().warn(String.format("No elements found by image [%s]", template));
return new ArrayList<>(0);
int matchCounter = (result.width() - template.width() + 1) * (result.height() - template.height() + 1);
List<Point> matchLocations = new ArrayList<>();
while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) {
matchCounter--;
Point matchLocation = minMaxLoc.maxLoc;
matchLocations.add(matchLocation);
Imgproc.rectangle(result, new Point(matchLocation.x, matchLocation.y), new Point(matchLocation.x + template.cols(),
matchLocation.y + template.rows()), new Scalar(0, 0, 0), -1);
minMaxLoc = Core.minMaxLoc(result);
}

return getElementsOnPoint(minMaxLoc.maxLoc, context);
return matchLocations.stream().map(matchLocation -> getElementOnPoint(matchLocation, context)).collect(Collectors.toList());
}

private List<WebElement> getElementsOnPoint(Point matchLocation, SearchContext context) {
int centerX = (int)(matchLocation.x + (template.width() / 2));
int centerY = (int)(matchLocation.y + (template.height() / 2));

JavascriptExecutor js;
if (!(context instanceof JavascriptExecutor)) {
AqualityServices.getLogger().debug("Current search context doesn't support executing scripts. " +
"Will take browser js executor instead");
js = AqualityServices.getBrowser().getDriver();
/**
* Gets a single element on point (find by center coordinates, then select closest to matchLocation).
*
* @param matchLocation location of the upper-left point of the element.
* @param context search context.
* If the searchContext is Locatable (like WebElement), adjust coordinates to be absolute coordinates.
* @return the closest found element.
*/
protected WebElement getElementOnPoint(Point matchLocation, SearchContext context) {
if (context instanceof Locatable) {
final org.openqa.selenium.Point point = ((Locatable) context).getCoordinates().onPage();
matchLocation.x += point.getX();
matchLocation.y += point.getY();
}
else {
js = (JavascriptExecutor) context;
}

int centerX = (int) (matchLocation.x + (template.width() / 2));
int centerY = (int) (matchLocation.y + (template.height() / 2));
//noinspection unchecked
return (List<WebElement>) js.executeScript("return document.elementsFromPoint(arguments[0], arguments[1]);", centerX, centerY);
List<WebElement> elements = (List<WebElement>) AqualityServices.getBrowser().executeScript("return document.elementsFromPoint(arguments[0], arguments[1]);", centerX, centerY);
elements.sort(Comparator.comparingDouble(e -> distanceToPoint(matchLocation, e)));
return elements.get(0);
}

/**
* Calculates distance from element to matching point.
*
* @param matchLocation matching point.
* @param element target element.
* @return distance in pixels.
*/
protected static double distanceToPoint(Point matchLocation, WebElement element) {
org.openqa.selenium.Point elementLocation = element.getLocation();
return Math.sqrt(Math.pow(matchLocation.x - elementLocation.x, 2) + Math.pow(matchLocation.y - elementLocation.y, 2));
}

private byte[] getScreenshot(SearchContext context) {
/**
* Takes screenshot from searchContext if supported, or from browser.
*
* @param context search context for element location.
* @return captured screenshot as byte array.
*/
protected byte[] getScreenshot(SearchContext context) {
byte[] screenshotBytes;

if (!(context instanceof TakesScreenshot)) {
Expand Down

0 comments on commit 4e82235

Please sign in to comment.