[Feature] Find elements by image

pom.xml

@@ -100,6 +100,12 @@
             <version>2.0.10</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.openpnp</groupId>
+            <artifactId>opencv</artifactId>
+            <version>[4.7.0,)</version>
+        </dependency>
+
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>

src/main/java/aquality/selenium/browser/JavaScript.java

@@ -21,7 +21,9 @@ public enum JavaScript {
     GET_CHECKBOX_STATE("getCheckBxState.js"),
     GET_COMBOBOX_SELECTED_TEXT("getCmbText.js"),
     GET_COMBOBOX_TEXTS("getCmbValues.js"),
+    GET_DEVICE_PIXEL_RATIO("getDevicePixelRatio.js"),
     GET_ELEMENT_BY_XPATH("getElementByXpath.js"),
+    GET_ELEMENTS_FROM_POINT("getElementsFromPoint.js"),
     GET_ELEMENT_CSS_SELECTOR("getElementCssSelector.js"),
     GET_ELEMENT_XPATH("getElementXPath.js"),
     GET_ELEMENT_TEXT("getElementText.js"),

src/main/java/aquality/selenium/elements/interfaces/ByImage.java

@@ -0,0 +1,180 @@
+package aquality.selenium.elements.interfaces;
+
+import aquality.selenium.browser.AqualityServices;
+import aquality.selenium.browser.JavaScript;
+import org.opencv.core.Point;
+import org.opencv.core.*;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+import org.openqa.selenium.*;
+import org.openqa.selenium.interactions.Locatable;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Locator to search elements by image.
+ * Takes screenshot and finds match using openCV.
+ * Performs screenshot scaling if devicePixelRatio != 1.
+ * Then finds elements by coordinates using javascript.
+ */
+public class ByImage extends By {
+    private static boolean wasLibraryLoaded = false;
+    private final Mat template;
+    private final String description;
+    private float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold();
+
+    private static void loadLibrary() {
+        if (!wasLibraryLoaded) {
+            nu.pattern.OpenCV.loadShared();
+            System.loadLibrary(org.opencv.core.Core.NATIVE_LIBRARY_NAME);
+            wasLibraryLoaded = true;
+        }
+    }
+
+    /**
+     * Constructor accepting image file.
+     *
+     * @param file image file to locate element by.
+     */
+    public ByImage(File file) {
+        loadLibrary();
+        description = file.getName();
+        this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED);
+    }
+
+    /**
+     * Constructor accepting image bytes.
+     *
+     * @param bytes image bytes to locate element by.
+     */
+    public ByImage(byte[] bytes) {
+        loadLibrary();
+        description = String.format("bytes[%d]", bytes.length);
+        this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED);
+    }
+
+    /**
+     * Sets threshold of image similarity.
+     * @param threshold a float between 0 and 1, where 1 means 100% match, and 0.5 means 50% match.
+     * @return current instance of ByImage locator.
+     */
+    public ByImage setThreshold(float threshold) {
+        if (threshold < 0 || threshold > 1) {
+            throw new IllegalArgumentException("Threshold must be a float between 0 and 1.");
+        }
+        this.threshold = threshold;
+        return this;
+    }
+
+    /**
+     * Gets threshold of image similarity.
+     * @return current value of threshold.
+     */
+    public float getThreshold() {
+        return threshold;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("ByImage: %s, size: (width:%d, height:%d)", description, template.width(), template.height());
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof ByImage)) {
+            return false;
+        }
+
+        ByImage that = (ByImage) o;
+
+        return this.template.equals(that.template);
+    }
+
+    @Override
+    public int hashCode() {
+        return template.hashCode();
+    }
+
+
+    @Override
+    public List<WebElement> findElements(SearchContext context) {
+        Mat source = getScreenshot(context);
+        Mat result = new Mat();
+        Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED);
+
+        Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result);
+
+        int matchCounter = Math.abs((result.width() - template.width() + 1) * (result.height() - template.height() + 1));
+        List<Point> matchLocations = new ArrayList<>();
+        while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) {
+            matchCounter--;
+            Point matchLocation = minMaxLoc.maxLoc;
+            matchLocations.add(matchLocation);
+            Imgproc.rectangle(result, new Point(matchLocation.x, matchLocation.y), new Point(matchLocation.x + template.cols(),
+                    matchLocation.y + template.rows()), new Scalar(0, 0, 0), -1);
+            minMaxLoc = Core.minMaxLoc(result);
+        }
+
+        return matchLocations.stream().map(matchLocation -> getElementOnPoint(matchLocation, context)).collect(Collectors.toList());
+    }
+
+    /**
+     * Gets a single element on point (find by center coordinates, then select closest to matchLocation).
+     *
+     * @param matchLocation location of the upper-left point of the element.
+     * @param context       search context.
+     *                      If the searchContext is Locatable (like WebElement), adjust coordinates to be absolute coordinates.
+     * @return the closest found element.
+     */
+    protected WebElement getElementOnPoint(Point matchLocation, SearchContext context) {
+        if (context instanceof Locatable) {
+            final org.openqa.selenium.Point point = ((Locatable) context).getCoordinates().onPage();
+            matchLocation.x += point.getX();
+            matchLocation.y += point.getY();
+        }
+        int centerX = (int) (matchLocation.x + (template.width() / 2));
+        int centerY = (int) (matchLocation.y + (template.height() / 2));
+        //noinspection unchecked
+        List<WebElement> elements = (List<WebElement>) AqualityServices.getBrowser()
+                .executeScript(JavaScript.GET_ELEMENTS_FROM_POINT, centerX, centerY);
+        elements.sort(Comparator.comparingDouble(e -> distanceToPoint(matchLocation, e)));
+        return elements.get(0);
+    }
+
+    /**
+     * Calculates distance from element to matching point.
+     *
+     * @param matchLocation matching point.
+     * @param element       target element.
+     * @return distance in pixels.
+     */
+    protected static double distanceToPoint(Point matchLocation, WebElement element) {
+        org.openqa.selenium.Point elementLocation = element.getLocation();
+        return Math.sqrt(Math.pow(matchLocation.x - elementLocation.x, 2) + Math.pow(matchLocation.y - elementLocation.y, 2));
+    }
+
+    /**
+     * Takes screenshot from searchContext if supported, or from browser.
+     *
+     * @param context search context for element location.
+     * @return captured screenshot as Mat object.
+     */
+    protected Mat getScreenshot(SearchContext context) {
+        byte[] screenshotBytes = context instanceof TakesScreenshot
+                ? ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES)
+                : AqualityServices.getBrowser().getScreenshot();
+        boolean isBrowserScreenshot = context instanceof WebDriver || !(context instanceof TakesScreenshot);
+        Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED);
+        long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO);
+        if (devicePixelRatio != 1 && isBrowserScreenshot) {
+            int scaledWidth = (int) (source.width() / devicePixelRatio);
+            int scaledHeight = (int) (source.height() / devicePixelRatio);
+            Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA);
+        }
+        return source;
+    }
+}

src/main/resources/js/getDevicePixelRatio.js

@@ -0,0 +1 @@
+return window.devicePixelRatio;

src/main/resources/js/getElementsFromPoint.js

@@ -0,0 +1 @@
+return document.elementsFromPoint(arguments[0], arguments[1]);

src/test/java/manytools/ManyToolsForm.java

@@ -1,9 +1,13 @@
 package manytools;
 
 import aquality.selenium.browser.AqualityServices;
+import aquality.selenium.core.utilities.IActionRetrier;
 import aquality.selenium.elements.interfaces.ILabel;
 import aquality.selenium.forms.Form;
 import org.openqa.selenium.By;
+import org.openqa.selenium.TimeoutException;
+
+import java.util.Collections;
 
 public abstract class ManyToolsForm<T extends ManyToolsForm<T>> extends Form {
     private static final String BASE_URL = "https://manytools.org/";
@@ -17,8 +21,10 @@ protected ManyToolsForm(String name) {
 
     @SuppressWarnings("unchecked")
     public T open() {
-        AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart());
-        AqualityServices.getBrowser().waitForPageToLoad();
+        AqualityServices.get(IActionRetrier.class).doWithRetry(() -> {
+            AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart());
+            AqualityServices.getBrowser().waitForPageToLoad();
+        }, Collections.singletonList(TimeoutException.class));
         return (T) this;
     }
 

src/test/java/tests/integration/LocatorTests.java

@@ -1,16 +1,23 @@
 package tests.integration;
 
+import aquality.selenium.browser.AqualityServices;
+import aquality.selenium.elements.interfaces.ByImage;
 import aquality.selenium.elements.interfaces.ILabel;
 import automationpractice.forms.ChallengingDomForm;
 import org.openqa.selenium.By;
+import org.openqa.selenium.OutputType;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.support.locators.RelativeLocator;
+import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 import org.testng.asserts.SoftAssert;
 import tests.BaseTest;
 import theinternet.TheInternetPage;
+import theinternet.forms.BrokenImagesForm;
+
 import java.util.List;
+
 import static aquality.selenium.locators.RelativeBySupplier.with;
 
 public class LocatorTests extends BaseTest {
@@ -26,6 +33,28 @@ public void beforeMethod() {
         navigate(TheInternetPage.CHALLENGING_DOM);
     }
 
+    @Test
+    public void testByImageLocator() {
+        BrokenImagesForm form = new BrokenImagesForm();
+        Assert.assertFalse(form.getLabelByImage().state().isDisplayed(), "Should be impossible to find element on page by image when it is absent");
+        getBrowser().goTo(form.getUrl());
+        Assert.assertTrue(form.getLabelByImage().state().isDisplayed(), "Should be possible to find element on page by image");
+        Assert.assertEquals(form.getLabelByImage().getElement().getTagName(), "img", "Correct element must be found");
+
+        List<ILabel> childLabels = form.getChildLabelsByImage();
+        List<ILabel> docLabels = form.getLabelsByImage();
+        Assert.assertTrue(docLabels.size() > 1, "List of elements should be possible to find by image");
+        Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count");
+
+        ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag");
+        float fullThreshold = 1;
+        ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)).setThreshold(fullThreshold),
+                "body screen");
+        Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot");
+        Assert.assertEquals(((ByImage)documentByImage.getLocator()).getThreshold(), fullThreshold, "Should be possible to get ByImage threshold");
+        Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found");
+    }
+
     @Test
     public void testAboveLocatorWithDifferentAboveParametersType() {
         ILabel cellInRow5Column5 = challengingDomForm.getCellInRow5Column5();

src/test/java/theinternet/forms/BrokenImagesForm.java

@@ -0,0 +1,33 @@
+package theinternet.forms;
+
+import aquality.selenium.elements.interfaces.ByImage;
+import aquality.selenium.elements.interfaces.ILabel;
+import org.openqa.selenium.By;
+import utils.FileUtil;
+
+import java.util.List;
+
+public class BrokenImagesForm extends TheInternetForm {
+    private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png"));
+
+    public BrokenImagesForm(){
+        super(By.id("content"), "Broken Images form");
+    }
+
+    public ILabel getLabelByImage(){
+        return getElementFactory().getLabel(imageLocator, "broken image");
+    }
+
+    public List<ILabel> getLabelsByImage(){
+        return getElementFactory().findElements(imageLocator, "broken image", ILabel.class);
+    }
+
+    public List<ILabel> getChildLabelsByImage(){
+        return getFormLabel().findChildElements(imageLocator, "broken image", ILabel.class);
+    }
+
+    @Override
+    protected String getUri() {
+        return "/broken_images";
+    }
+}