diff --git a/pom.xml b/pom.xml
index ac2724c..2539ac5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,6 +100,12 @@
2.0.10
test
+
+ org.openpnp
+ opencv
+ [4.7.0,)
+
+
org.testng
testng
diff --git a/src/main/java/aquality/selenium/browser/JavaScript.java b/src/main/java/aquality/selenium/browser/JavaScript.java
index b079794..f3825c3 100644
--- a/src/main/java/aquality/selenium/browser/JavaScript.java
+++ b/src/main/java/aquality/selenium/browser/JavaScript.java
@@ -21,7 +21,9 @@ public enum JavaScript {
GET_CHECKBOX_STATE("getCheckBxState.js"),
GET_COMBOBOX_SELECTED_TEXT("getCmbText.js"),
GET_COMBOBOX_TEXTS("getCmbValues.js"),
+ GET_DEVICE_PIXEL_RATIO("getDevicePixelRatio.js"),
GET_ELEMENT_BY_XPATH("getElementByXpath.js"),
+ GET_ELEMENTS_FROM_POINT("getElementsFromPoint.js"),
GET_ELEMENT_CSS_SELECTOR("getElementCssSelector.js"),
GET_ELEMENT_XPATH("getElementXPath.js"),
GET_ELEMENT_TEXT("getElementText.js"),
diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java
new file mode 100644
index 0000000..7a8af48
--- /dev/null
+++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java
@@ -0,0 +1,180 @@
+package aquality.selenium.elements.interfaces;
+
+import aquality.selenium.browser.AqualityServices;
+import aquality.selenium.browser.JavaScript;
+import org.opencv.core.Point;
+import org.opencv.core.*;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+import org.openqa.selenium.*;
+import org.openqa.selenium.interactions.Locatable;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Locator to search elements by image.
+ * Takes screenshot and finds match using openCV.
+ * Performs screenshot scaling if devicePixelRatio != 1.
+ * Then finds elements by coordinates using javascript.
+ */
+public class ByImage extends By {
+ private static boolean wasLibraryLoaded = false;
+ private final Mat template;
+ private final String description;
+ private float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold();
+
+ private static void loadLibrary() {
+ if (!wasLibraryLoaded) {
+ nu.pattern.OpenCV.loadShared();
+ System.loadLibrary(org.opencv.core.Core.NATIVE_LIBRARY_NAME);
+ wasLibraryLoaded = true;
+ }
+ }
+
+ /**
+ * Constructor accepting image file.
+ *
+ * @param file image file to locate element by.
+ */
+ public ByImage(File file) {
+ loadLibrary();
+ description = file.getName();
+ this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED);
+ }
+
+ /**
+ * Constructor accepting image bytes.
+ *
+ * @param bytes image bytes to locate element by.
+ */
+ public ByImage(byte[] bytes) {
+ loadLibrary();
+ description = String.format("bytes[%d]", bytes.length);
+ this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED);
+ }
+
+ /**
+ * Sets threshold of image similarity.
+ * @param threshold a float between 0 and 1, where 1 means 100% match, and 0.5 means 50% match.
+ * @return current instance of ByImage locator.
+ */
+ public ByImage setThreshold(float threshold) {
+ if (threshold < 0 || threshold > 1) {
+ throw new IllegalArgumentException("Threshold must be a float between 0 and 1.");
+ }
+ this.threshold = threshold;
+ return this;
+ }
+
+ /**
+ * Gets threshold of image similarity.
+ * @return current value of threshold.
+ */
+ public float getThreshold() {
+ return threshold;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("ByImage: %s, size: (width:%d, height:%d)", description, template.width(), template.height());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ByImage)) {
+ return false;
+ }
+
+ ByImage that = (ByImage) o;
+
+ return this.template.equals(that.template);
+ }
+
+ @Override
+ public int hashCode() {
+ return template.hashCode();
+ }
+
+
+ @Override
+ public List findElements(SearchContext context) {
+ Mat source = getScreenshot(context);
+ Mat result = new Mat();
+ Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED);
+
+ Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result);
+
+ int matchCounter = Math.abs((result.width() - template.width() + 1) * (result.height() - template.height() + 1));
+ List matchLocations = new ArrayList<>();
+ while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) {
+ matchCounter--;
+ Point matchLocation = minMaxLoc.maxLoc;
+ matchLocations.add(matchLocation);
+ Imgproc.rectangle(result, new Point(matchLocation.x, matchLocation.y), new Point(matchLocation.x + template.cols(),
+ matchLocation.y + template.rows()), new Scalar(0, 0, 0), -1);
+ minMaxLoc = Core.minMaxLoc(result);
+ }
+
+ return matchLocations.stream().map(matchLocation -> getElementOnPoint(matchLocation, context)).collect(Collectors.toList());
+ }
+
+ /**
+ * Gets a single element on point (find by center coordinates, then select closest to matchLocation).
+ *
+ * @param matchLocation location of the upper-left point of the element.
+ * @param context search context.
+ * If the searchContext is Locatable (like WebElement), adjust coordinates to be absolute coordinates.
+ * @return the closest found element.
+ */
+ protected WebElement getElementOnPoint(Point matchLocation, SearchContext context) {
+ if (context instanceof Locatable) {
+ final org.openqa.selenium.Point point = ((Locatable) context).getCoordinates().onPage();
+ matchLocation.x += point.getX();
+ matchLocation.y += point.getY();
+ }
+ int centerX = (int) (matchLocation.x + (template.width() / 2));
+ int centerY = (int) (matchLocation.y + (template.height() / 2));
+ //noinspection unchecked
+ List elements = (List) AqualityServices.getBrowser()
+ .executeScript(JavaScript.GET_ELEMENTS_FROM_POINT, centerX, centerY);
+ elements.sort(Comparator.comparingDouble(e -> distanceToPoint(matchLocation, e)));
+ return elements.get(0);
+ }
+
+ /**
+ * Calculates distance from element to matching point.
+ *
+ * @param matchLocation matching point.
+ * @param element target element.
+ * @return distance in pixels.
+ */
+ protected static double distanceToPoint(Point matchLocation, WebElement element) {
+ org.openqa.selenium.Point elementLocation = element.getLocation();
+ return Math.sqrt(Math.pow(matchLocation.x - elementLocation.x, 2) + Math.pow(matchLocation.y - elementLocation.y, 2));
+ }
+
+ /**
+ * Takes screenshot from searchContext if supported, or from browser.
+ *
+ * @param context search context for element location.
+ * @return captured screenshot as Mat object.
+ */
+ protected Mat getScreenshot(SearchContext context) {
+ byte[] screenshotBytes = context instanceof TakesScreenshot
+ ? ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES)
+ : AqualityServices.getBrowser().getScreenshot();
+ boolean isBrowserScreenshot = context instanceof WebDriver || !(context instanceof TakesScreenshot);
+ Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED);
+ long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO);
+ if (devicePixelRatio != 1 && isBrowserScreenshot) {
+ int scaledWidth = (int) (source.width() / devicePixelRatio);
+ int scaledHeight = (int) (source.height() / devicePixelRatio);
+ Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA);
+ }
+ return source;
+ }
+}
diff --git a/src/main/resources/js/getDevicePixelRatio.js b/src/main/resources/js/getDevicePixelRatio.js
new file mode 100644
index 0000000..679d44c
--- /dev/null
+++ b/src/main/resources/js/getDevicePixelRatio.js
@@ -0,0 +1 @@
+return window.devicePixelRatio;
diff --git a/src/main/resources/js/getElementsFromPoint.js b/src/main/resources/js/getElementsFromPoint.js
new file mode 100644
index 0000000..7739565
--- /dev/null
+++ b/src/main/resources/js/getElementsFromPoint.js
@@ -0,0 +1 @@
+return document.elementsFromPoint(arguments[0], arguments[1]);
diff --git a/src/test/java/manytools/ManyToolsForm.java b/src/test/java/manytools/ManyToolsForm.java
index 5eda112..2849ccf 100644
--- a/src/test/java/manytools/ManyToolsForm.java
+++ b/src/test/java/manytools/ManyToolsForm.java
@@ -1,9 +1,13 @@
package manytools;
import aquality.selenium.browser.AqualityServices;
+import aquality.selenium.core.utilities.IActionRetrier;
import aquality.selenium.elements.interfaces.ILabel;
import aquality.selenium.forms.Form;
import org.openqa.selenium.By;
+import org.openqa.selenium.TimeoutException;
+
+import java.util.Collections;
public abstract class ManyToolsForm> extends Form {
private static final String BASE_URL = "https://manytools.org/";
@@ -17,8 +21,10 @@ protected ManyToolsForm(String name) {
@SuppressWarnings("unchecked")
public T open() {
- AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart());
- AqualityServices.getBrowser().waitForPageToLoad();
+ AqualityServices.get(IActionRetrier.class).doWithRetry(() -> {
+ AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart());
+ AqualityServices.getBrowser().waitForPageToLoad();
+ }, Collections.singletonList(TimeoutException.class));
return (T) this;
}
diff --git a/src/test/java/tests/integration/LocatorTests.java b/src/test/java/tests/integration/LocatorTests.java
index b2b661c..43e92e6 100644
--- a/src/test/java/tests/integration/LocatorTests.java
+++ b/src/test/java/tests/integration/LocatorTests.java
@@ -1,16 +1,23 @@
package tests.integration;
+import aquality.selenium.browser.AqualityServices;
+import aquality.selenium.elements.interfaces.ByImage;
import aquality.selenium.elements.interfaces.ILabel;
import automationpractice.forms.ChallengingDomForm;
import org.openqa.selenium.By;
+import org.openqa.selenium.OutputType;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.locators.RelativeLocator;
+import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import org.testng.asserts.SoftAssert;
import tests.BaseTest;
import theinternet.TheInternetPage;
+import theinternet.forms.BrokenImagesForm;
+
import java.util.List;
+
import static aquality.selenium.locators.RelativeBySupplier.with;
public class LocatorTests extends BaseTest {
@@ -26,6 +33,28 @@ public void beforeMethod() {
navigate(TheInternetPage.CHALLENGING_DOM);
}
+ @Test
+ public void testByImageLocator() {
+ BrokenImagesForm form = new BrokenImagesForm();
+ Assert.assertFalse(form.getLabelByImage().state().isDisplayed(), "Should be impossible to find element on page by image when it is absent");
+ getBrowser().goTo(form.getUrl());
+ Assert.assertTrue(form.getLabelByImage().state().isDisplayed(), "Should be possible to find element on page by image");
+ Assert.assertEquals(form.getLabelByImage().getElement().getTagName(), "img", "Correct element must be found");
+
+ List childLabels = form.getChildLabelsByImage();
+ List docLabels = form.getLabelsByImage();
+ Assert.assertTrue(docLabels.size() > 1, "List of elements should be possible to find by image");
+ Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count");
+
+ ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag");
+ float fullThreshold = 1;
+ ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)).setThreshold(fullThreshold),
+ "body screen");
+ Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot");
+ Assert.assertEquals(((ByImage)documentByImage.getLocator()).getThreshold(), fullThreshold, "Should be possible to get ByImage threshold");
+ Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found");
+ }
+
@Test
public void testAboveLocatorWithDifferentAboveParametersType() {
ILabel cellInRow5Column5 = challengingDomForm.getCellInRow5Column5();
diff --git a/src/test/java/theinternet/forms/BrokenImagesForm.java b/src/test/java/theinternet/forms/BrokenImagesForm.java
new file mode 100644
index 0000000..9cffb24
--- /dev/null
+++ b/src/test/java/theinternet/forms/BrokenImagesForm.java
@@ -0,0 +1,33 @@
+package theinternet.forms;
+
+import aquality.selenium.elements.interfaces.ByImage;
+import aquality.selenium.elements.interfaces.ILabel;
+import org.openqa.selenium.By;
+import utils.FileUtil;
+
+import java.util.List;
+
+public class BrokenImagesForm extends TheInternetForm {
+ private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png"));
+
+ public BrokenImagesForm(){
+ super(By.id("content"), "Broken Images form");
+ }
+
+ public ILabel getLabelByImage(){
+ return getElementFactory().getLabel(imageLocator, "broken image");
+ }
+
+ public List getLabelsByImage(){
+ return getElementFactory().findElements(imageLocator, "broken image", ILabel.class);
+ }
+
+ public List getChildLabelsByImage(){
+ return getFormLabel().findChildElements(imageLocator, "broken image", ILabel.class);
+ }
+
+ @Override
+ protected String getUri() {
+ return "/broken_images";
+ }
+}
diff --git a/src/test/resources/brokenImage.png b/src/test/resources/brokenImage.png
new file mode 100644
index 0000000..9edd76f
Binary files /dev/null and b/src/test/resources/brokenImage.png differ