diff --git a/pom.xml b/pom.xml index ac2724c..2539ac5 100644 --- a/pom.xml +++ b/pom.xml @@ -100,6 +100,12 @@ 2.0.10 test + + org.openpnp + opencv + [4.7.0,) + + org.testng testng diff --git a/src/main/java/aquality/selenium/browser/JavaScript.java b/src/main/java/aquality/selenium/browser/JavaScript.java index b079794..f3825c3 100644 --- a/src/main/java/aquality/selenium/browser/JavaScript.java +++ b/src/main/java/aquality/selenium/browser/JavaScript.java @@ -21,7 +21,9 @@ public enum JavaScript { GET_CHECKBOX_STATE("getCheckBxState.js"), GET_COMBOBOX_SELECTED_TEXT("getCmbText.js"), GET_COMBOBOX_TEXTS("getCmbValues.js"), + GET_DEVICE_PIXEL_RATIO("getDevicePixelRatio.js"), GET_ELEMENT_BY_XPATH("getElementByXpath.js"), + GET_ELEMENTS_FROM_POINT("getElementsFromPoint.js"), GET_ELEMENT_CSS_SELECTOR("getElementCssSelector.js"), GET_ELEMENT_XPATH("getElementXPath.js"), GET_ELEMENT_TEXT("getElementText.js"), diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java new file mode 100644 index 0000000..7a8af48 --- /dev/null +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -0,0 +1,180 @@ +package aquality.selenium.elements.interfaces; + +import aquality.selenium.browser.AqualityServices; +import aquality.selenium.browser.JavaScript; +import org.opencv.core.Point; +import org.opencv.core.*; +import org.opencv.imgcodecs.Imgcodecs; +import org.opencv.imgproc.Imgproc; +import org.openqa.selenium.*; +import org.openqa.selenium.interactions.Locatable; + +import java.io.File; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Locator to search elements by image. + * Takes screenshot and finds match using openCV. + * Performs screenshot scaling if devicePixelRatio != 1. + * Then finds elements by coordinates using javascript. + */ +public class ByImage extends By { + private static boolean wasLibraryLoaded = false; + private final Mat template; + private final String description; + private float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold(); + + private static void loadLibrary() { + if (!wasLibraryLoaded) { + nu.pattern.OpenCV.loadShared(); + System.loadLibrary(org.opencv.core.Core.NATIVE_LIBRARY_NAME); + wasLibraryLoaded = true; + } + } + + /** + * Constructor accepting image file. + * + * @param file image file to locate element by. + */ + public ByImage(File file) { + loadLibrary(); + description = file.getName(); + this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED); + } + + /** + * Constructor accepting image bytes. + * + * @param bytes image bytes to locate element by. + */ + public ByImage(byte[] bytes) { + loadLibrary(); + description = String.format("bytes[%d]", bytes.length); + this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); + } + + /** + * Sets threshold of image similarity. + * @param threshold a float between 0 and 1, where 1 means 100% match, and 0.5 means 50% match. + * @return current instance of ByImage locator. + */ + public ByImage setThreshold(float threshold) { + if (threshold < 0 || threshold > 1) { + throw new IllegalArgumentException("Threshold must be a float between 0 and 1."); + } + this.threshold = threshold; + return this; + } + + /** + * Gets threshold of image similarity. + * @return current value of threshold. + */ + public float getThreshold() { + return threshold; + } + + @Override + public String toString() { + return String.format("ByImage: %s, size: (width:%d, height:%d)", description, template.width(), template.height()); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ByImage)) { + return false; + } + + ByImage that = (ByImage) o; + + return this.template.equals(that.template); + } + + @Override + public int hashCode() { + return template.hashCode(); + } + + + @Override + public List findElements(SearchContext context) { + Mat source = getScreenshot(context); + Mat result = new Mat(); + Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED); + + Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result); + + int matchCounter = Math.abs((result.width() - template.width() + 1) * (result.height() - template.height() + 1)); + List matchLocations = new ArrayList<>(); + while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) { + matchCounter--; + Point matchLocation = minMaxLoc.maxLoc; + matchLocations.add(matchLocation); + Imgproc.rectangle(result, new Point(matchLocation.x, matchLocation.y), new Point(matchLocation.x + template.cols(), + matchLocation.y + template.rows()), new Scalar(0, 0, 0), -1); + minMaxLoc = Core.minMaxLoc(result); + } + + return matchLocations.stream().map(matchLocation -> getElementOnPoint(matchLocation, context)).collect(Collectors.toList()); + } + + /** + * Gets a single element on point (find by center coordinates, then select closest to matchLocation). + * + * @param matchLocation location of the upper-left point of the element. + * @param context search context. + * If the searchContext is Locatable (like WebElement), adjust coordinates to be absolute coordinates. + * @return the closest found element. + */ + protected WebElement getElementOnPoint(Point matchLocation, SearchContext context) { + if (context instanceof Locatable) { + final org.openqa.selenium.Point point = ((Locatable) context).getCoordinates().onPage(); + matchLocation.x += point.getX(); + matchLocation.y += point.getY(); + } + int centerX = (int) (matchLocation.x + (template.width() / 2)); + int centerY = (int) (matchLocation.y + (template.height() / 2)); + //noinspection unchecked + List elements = (List) AqualityServices.getBrowser() + .executeScript(JavaScript.GET_ELEMENTS_FROM_POINT, centerX, centerY); + elements.sort(Comparator.comparingDouble(e -> distanceToPoint(matchLocation, e))); + return elements.get(0); + } + + /** + * Calculates distance from element to matching point. + * + * @param matchLocation matching point. + * @param element target element. + * @return distance in pixels. + */ + protected static double distanceToPoint(Point matchLocation, WebElement element) { + org.openqa.selenium.Point elementLocation = element.getLocation(); + return Math.sqrt(Math.pow(matchLocation.x - elementLocation.x, 2) + Math.pow(matchLocation.y - elementLocation.y, 2)); + } + + /** + * Takes screenshot from searchContext if supported, or from browser. + * + * @param context search context for element location. + * @return captured screenshot as Mat object. + */ + protected Mat getScreenshot(SearchContext context) { + byte[] screenshotBytes = context instanceof TakesScreenshot + ? ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES) + : AqualityServices.getBrowser().getScreenshot(); + boolean isBrowserScreenshot = context instanceof WebDriver || !(context instanceof TakesScreenshot); + Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED); + long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO); + if (devicePixelRatio != 1 && isBrowserScreenshot) { + int scaledWidth = (int) (source.width() / devicePixelRatio); + int scaledHeight = (int) (source.height() / devicePixelRatio); + Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA); + } + return source; + } +} diff --git a/src/main/resources/js/getDevicePixelRatio.js b/src/main/resources/js/getDevicePixelRatio.js new file mode 100644 index 0000000..679d44c --- /dev/null +++ b/src/main/resources/js/getDevicePixelRatio.js @@ -0,0 +1 @@ +return window.devicePixelRatio; diff --git a/src/main/resources/js/getElementsFromPoint.js b/src/main/resources/js/getElementsFromPoint.js new file mode 100644 index 0000000..7739565 --- /dev/null +++ b/src/main/resources/js/getElementsFromPoint.js @@ -0,0 +1 @@ +return document.elementsFromPoint(arguments[0], arguments[1]); diff --git a/src/test/java/manytools/ManyToolsForm.java b/src/test/java/manytools/ManyToolsForm.java index 5eda112..2849ccf 100644 --- a/src/test/java/manytools/ManyToolsForm.java +++ b/src/test/java/manytools/ManyToolsForm.java @@ -1,9 +1,13 @@ package manytools; import aquality.selenium.browser.AqualityServices; +import aquality.selenium.core.utilities.IActionRetrier; import aquality.selenium.elements.interfaces.ILabel; import aquality.selenium.forms.Form; import org.openqa.selenium.By; +import org.openqa.selenium.TimeoutException; + +import java.util.Collections; public abstract class ManyToolsForm> extends Form { private static final String BASE_URL = "https://manytools.org/"; @@ -17,8 +21,10 @@ protected ManyToolsForm(String name) { @SuppressWarnings("unchecked") public T open() { - AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart()); - AqualityServices.getBrowser().waitForPageToLoad(); + AqualityServices.get(IActionRetrier.class).doWithRetry(() -> { + AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart()); + AqualityServices.getBrowser().waitForPageToLoad(); + }, Collections.singletonList(TimeoutException.class)); return (T) this; } diff --git a/src/test/java/tests/integration/LocatorTests.java b/src/test/java/tests/integration/LocatorTests.java index b2b661c..43e92e6 100644 --- a/src/test/java/tests/integration/LocatorTests.java +++ b/src/test/java/tests/integration/LocatorTests.java @@ -1,16 +1,23 @@ package tests.integration; +import aquality.selenium.browser.AqualityServices; +import aquality.selenium.elements.interfaces.ByImage; import aquality.selenium.elements.interfaces.ILabel; import automationpractice.forms.ChallengingDomForm; import org.openqa.selenium.By; +import org.openqa.selenium.OutputType; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.locators.RelativeLocator; +import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.testng.asserts.SoftAssert; import tests.BaseTest; import theinternet.TheInternetPage; +import theinternet.forms.BrokenImagesForm; + import java.util.List; + import static aquality.selenium.locators.RelativeBySupplier.with; public class LocatorTests extends BaseTest { @@ -26,6 +33,28 @@ public void beforeMethod() { navigate(TheInternetPage.CHALLENGING_DOM); } + @Test + public void testByImageLocator() { + BrokenImagesForm form = new BrokenImagesForm(); + Assert.assertFalse(form.getLabelByImage().state().isDisplayed(), "Should be impossible to find element on page by image when it is absent"); + getBrowser().goTo(form.getUrl()); + Assert.assertTrue(form.getLabelByImage().state().isDisplayed(), "Should be possible to find element on page by image"); + Assert.assertEquals(form.getLabelByImage().getElement().getTagName(), "img", "Correct element must be found"); + + List childLabels = form.getChildLabelsByImage(); + List docLabels = form.getLabelsByImage(); + Assert.assertTrue(docLabels.size() > 1, "List of elements should be possible to find by image"); + Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count"); + + ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag"); + float fullThreshold = 1; + ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)).setThreshold(fullThreshold), + "body screen"); + Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot"); + Assert.assertEquals(((ByImage)documentByImage.getLocator()).getThreshold(), fullThreshold, "Should be possible to get ByImage threshold"); + Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found"); + } + @Test public void testAboveLocatorWithDifferentAboveParametersType() { ILabel cellInRow5Column5 = challengingDomForm.getCellInRow5Column5(); diff --git a/src/test/java/theinternet/forms/BrokenImagesForm.java b/src/test/java/theinternet/forms/BrokenImagesForm.java new file mode 100644 index 0000000..9cffb24 --- /dev/null +++ b/src/test/java/theinternet/forms/BrokenImagesForm.java @@ -0,0 +1,33 @@ +package theinternet.forms; + +import aquality.selenium.elements.interfaces.ByImage; +import aquality.selenium.elements.interfaces.ILabel; +import org.openqa.selenium.By; +import utils.FileUtil; + +import java.util.List; + +public class BrokenImagesForm extends TheInternetForm { + private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png")); + + public BrokenImagesForm(){ + super(By.id("content"), "Broken Images form"); + } + + public ILabel getLabelByImage(){ + return getElementFactory().getLabel(imageLocator, "broken image"); + } + + public List getLabelsByImage(){ + return getElementFactory().findElements(imageLocator, "broken image", ILabel.class); + } + + public List getChildLabelsByImage(){ + return getFormLabel().findChildElements(imageLocator, "broken image", ILabel.class); + } + + @Override + protected String getUri() { + return "/broken_images"; + } +} diff --git a/src/test/resources/brokenImage.png b/src/test/resources/brokenImage.png new file mode 100644 index 0000000..9edd76f Binary files /dev/null and b/src/test/resources/brokenImage.png differ