From 3dcf029def5dd13d89b9b3cce2e0455b634591bf Mon Sep 17 00:00:00 2001 From: Aleksey2 Meleshko Date: Mon, 24 Apr 2023 21:33:50 +0200 Subject: [PATCH 1/9] [WIP] Implement ByImage locator --- pom.xml | 6 ++ .../selenium/elements/interfaces/ByImage.java | 87 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 src/main/java/aquality/selenium/elements/interfaces/ByImage.java diff --git a/pom.xml b/pom.xml index 6547771..adfdc30 100644 --- a/pom.xml +++ b/pom.xml @@ -96,6 +96,12 @@ 2.14.2 + + org.openpnp + opencv + [4.7.0,) + + org.testng testng diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java new file mode 100644 index 0000000..ab75e8f --- /dev/null +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -0,0 +1,87 @@ +package aquality.selenium.elements.interfaces; + +import aquality.selenium.browser.AqualityServices; +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.MatOfByte; +import org.opencv.core.Point; +import org.opencv.imgcodecs.Imgcodecs; +import org.opencv.imgproc.Imgproc; +import org.openqa.selenium.*; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class ByImage extends By { + private static boolean wasLibraryLoaded = false; + private final Mat template; + + private static void loadLibrary() { + if (!wasLibraryLoaded) { + nu.pattern.OpenCV.loadShared(); + System.loadLibrary(org.opencv.core.Core.NATIVE_LIBRARY_NAME); + wasLibraryLoaded = true; + } + } + + public ByImage(File file) { + loadLibrary(); + this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED); + } + + public ByImage(byte[] bytes) { + loadLibrary(); + this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); + } + + @Override + public List findElements(SearchContext context) { + byte[] screenshotBytes = getScreenshot(context); + Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED); + Mat result = new Mat(); + Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED); + + float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold(); + Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result); + + if (minMaxLoc.maxVal < threshold) { + AqualityServices.getLogger().warn(String.format("No elements found by image [%s]", template)); + return new ArrayList<>(0); + } + + return getElementsOnPoint(minMaxLoc.maxLoc, context); + } + + private List getElementsOnPoint(Point matchLocation, SearchContext context) { + int centerX = (int)(matchLocation.x + (template.width() / 2)); + int centerY = (int)(matchLocation.y + (template.height() / 2)); + + JavascriptExecutor js; + if (!(context instanceof JavascriptExecutor)) { + AqualityServices.getLogger().debug("Current search context doesn't support executing scripts. " + + "Will take browser js executor instead"); + js = AqualityServices.getBrowser().getDriver(); + } + else { + js = (JavascriptExecutor) context; + } + + //noinspection unchecked + return (List) js.executeScript("return document.elementsFromPoint(arguments[0], arguments[1]);", centerX, centerY); + } + + private byte[] getScreenshot(SearchContext context) { + byte[] screenshotBytes; + + if (!(context instanceof TakesScreenshot)) { + AqualityServices.getLogger().debug("Current search context doesn't support taking screenshots. " + + "Will take browser screenshot instead"); + screenshotBytes = AqualityServices.getBrowser().getScreenshot(); + } else { + screenshotBytes = ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES); + } + + return screenshotBytes; + } +} From 4e822353118a87a587a915a5c5010c047103343d Mon Sep 17 00:00:00 2001 From: Aleksey2 Meleshko Date: Tue, 25 Apr 2023 15:52:12 +0200 Subject: [PATCH 2/9] [WIP] Rework elements location strategy: - Finding the closest element to matching point instead of the topmost element/ all elements on point - Support finding multiple elements (multiple image matches) - Support relative search (e.g. from element) - Add javadocs --- .../selenium/elements/interfaces/ByImage.java | 93 ++++++++++++++----- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java index ab75e8f..6e6428a 100644 --- a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -1,18 +1,24 @@ package aquality.selenium.elements.interfaces; import aquality.selenium.browser.AqualityServices; -import org.opencv.core.Core; -import org.opencv.core.Mat; -import org.opencv.core.MatOfByte; import org.opencv.core.Point; +import org.opencv.core.*; import org.opencv.imgcodecs.Imgcodecs; import org.opencv.imgproc.Imgproc; import org.openqa.selenium.*; +import org.openqa.selenium.interactions.Locatable; import java.io.File; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; +/** + * Locator to search elements by image. + * Takes screenshot and finds match using openCV. + * Then finds elements by coordinates using javascript. + */ public class ByImage extends By { private static boolean wasLibraryLoaded = false; private final Mat template; @@ -25,16 +31,31 @@ private static void loadLibrary() { } } + /** + * Constructor accepting image file. + * + * @param file image file to locate element by. + */ public ByImage(File file) { loadLibrary(); this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED); } + /** + * Constructor accepting image file. + * + * @param bytes image bytes to locate element by. + */ public ByImage(byte[] bytes) { loadLibrary(); this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); } + @Override + public String toString() { + return "ByImage: " + new Dimension(template.width(), template.height()); + } + @Override public List findElements(SearchContext context) { byte[] screenshotBytes = getScreenshot(context); @@ -45,33 +66,61 @@ public List findElements(SearchContext context) { float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold(); Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result); - if (minMaxLoc.maxVal < threshold) { - AqualityServices.getLogger().warn(String.format("No elements found by image [%s]", template)); - return new ArrayList<>(0); + int matchCounter = (result.width() - template.width() + 1) * (result.height() - template.height() + 1); + List matchLocations = new ArrayList<>(); + while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) { + matchCounter--; + Point matchLocation = minMaxLoc.maxLoc; + matchLocations.add(matchLocation); + Imgproc.rectangle(result, new Point(matchLocation.x, matchLocation.y), new Point(matchLocation.x + template.cols(), + matchLocation.y + template.rows()), new Scalar(0, 0, 0), -1); + minMaxLoc = Core.minMaxLoc(result); } - return getElementsOnPoint(minMaxLoc.maxLoc, context); + return matchLocations.stream().map(matchLocation -> getElementOnPoint(matchLocation, context)).collect(Collectors.toList()); } - private List getElementsOnPoint(Point matchLocation, SearchContext context) { - int centerX = (int)(matchLocation.x + (template.width() / 2)); - int centerY = (int)(matchLocation.y + (template.height() / 2)); - - JavascriptExecutor js; - if (!(context instanceof JavascriptExecutor)) { - AqualityServices.getLogger().debug("Current search context doesn't support executing scripts. " + - "Will take browser js executor instead"); - js = AqualityServices.getBrowser().getDriver(); + /** + * Gets a single element on point (find by center coordinates, then select closest to matchLocation). + * + * @param matchLocation location of the upper-left point of the element. + * @param context search context. + * If the searchContext is Locatable (like WebElement), adjust coordinates to be absolute coordinates. + * @return the closest found element. + */ + protected WebElement getElementOnPoint(Point matchLocation, SearchContext context) { + if (context instanceof Locatable) { + final org.openqa.selenium.Point point = ((Locatable) context).getCoordinates().onPage(); + matchLocation.x += point.getX(); + matchLocation.y += point.getY(); } - else { - js = (JavascriptExecutor) context; - } - + int centerX = (int) (matchLocation.x + (template.width() / 2)); + int centerY = (int) (matchLocation.y + (template.height() / 2)); //noinspection unchecked - return (List) js.executeScript("return document.elementsFromPoint(arguments[0], arguments[1]);", centerX, centerY); + List elements = (List) AqualityServices.getBrowser().executeScript("return document.elementsFromPoint(arguments[0], arguments[1]);", centerX, centerY); + elements.sort(Comparator.comparingDouble(e -> distanceToPoint(matchLocation, e))); + return elements.get(0); + } + + /** + * Calculates distance from element to matching point. + * + * @param matchLocation matching point. + * @param element target element. + * @return distance in pixels. + */ + protected static double distanceToPoint(Point matchLocation, WebElement element) { + org.openqa.selenium.Point elementLocation = element.getLocation(); + return Math.sqrt(Math.pow(matchLocation.x - elementLocation.x, 2) + Math.pow(matchLocation.y - elementLocation.y, 2)); } - private byte[] getScreenshot(SearchContext context) { + /** + * Takes screenshot from searchContext if supported, or from browser. + * + * @param context search context for element location. + * @return captured screenshot as byte array. + */ + protected byte[] getScreenshot(SearchContext context) { byte[] screenshotBytes; if (!(context instanceof TakesScreenshot)) { From bdbddefa833e9bc1c76354816b106226ee1bd959 Mon Sep 17 00:00:00 2001 From: Aleksey2 Meleshko Date: Tue, 25 Apr 2023 21:22:20 +0200 Subject: [PATCH 3/9] - move js to separate file - simplify getScreenshot method, remove debug logging - add locator test --- .../aquality/selenium/browser/JavaScript.java | 1 + .../selenium/elements/interfaces/ByImage.java | 18 ++++------ src/main/resources/js/getElementsFromPoint.js | 1 + .../java/tests/integration/LocatorTests.java | 22 ++++++++++++ .../theinternet/forms/BrokenImagesForm.java | 33 ++++++++++++++++++ src/test/resources/brokenImage.png | Bin 0 -> 765 bytes 6 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 src/main/resources/js/getElementsFromPoint.js create mode 100644 src/test/java/theinternet/forms/BrokenImagesForm.java create mode 100644 src/test/resources/brokenImage.png diff --git a/src/main/java/aquality/selenium/browser/JavaScript.java b/src/main/java/aquality/selenium/browser/JavaScript.java index b9a4c19..f8f13d0 100644 --- a/src/main/java/aquality/selenium/browser/JavaScript.java +++ b/src/main/java/aquality/selenium/browser/JavaScript.java @@ -22,6 +22,7 @@ public enum JavaScript { GET_COMBOBOX_SELECTED_TEXT("getCmbText.js"), GET_COMBOBOX_TEXTS("getCmbValues.js"), GET_ELEMENT_BY_XPATH("getElementByXpath.js"), + GET_ELEMENTS_FROM_POINT("getElementsFromPoint.js"), GET_ELEMENT_XPATH("getElementXPath.js"), GET_ELEMENT_TEXT("getElementText.js"), GET_TEXT_FIRST_CHILD("getTextFirstChild.js"), diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java index 6e6428a..b7ccc8f 100644 --- a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -1,6 +1,7 @@ package aquality.selenium.elements.interfaces; import aquality.selenium.browser.AqualityServices; +import aquality.selenium.browser.JavaScript; import org.opencv.core.Point; import org.opencv.core.*; import org.opencv.imgcodecs.Imgcodecs; @@ -97,7 +98,8 @@ protected WebElement getElementOnPoint(Point matchLocation, SearchContext contex int centerX = (int) (matchLocation.x + (template.width() / 2)); int centerY = (int) (matchLocation.y + (template.height() / 2)); //noinspection unchecked - List elements = (List) AqualityServices.getBrowser().executeScript("return document.elementsFromPoint(arguments[0], arguments[1]);", centerX, centerY); + List elements = (List) AqualityServices.getBrowser() + .executeScript(JavaScript.GET_ELEMENTS_FROM_POINT, centerX, centerY); elements.sort(Comparator.comparingDouble(e -> distanceToPoint(matchLocation, e))); return elements.get(0); } @@ -121,16 +123,8 @@ protected static double distanceToPoint(Point matchLocation, WebElement element) * @return captured screenshot as byte array. */ protected byte[] getScreenshot(SearchContext context) { - byte[] screenshotBytes; - - if (!(context instanceof TakesScreenshot)) { - AqualityServices.getLogger().debug("Current search context doesn't support taking screenshots. " + - "Will take browser screenshot instead"); - screenshotBytes = AqualityServices.getBrowser().getScreenshot(); - } else { - screenshotBytes = ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES); - } - - return screenshotBytes; + return !(context instanceof TakesScreenshot) + ? AqualityServices.getBrowser().getScreenshot() + : ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES); } } diff --git a/src/main/resources/js/getElementsFromPoint.js b/src/main/resources/js/getElementsFromPoint.js new file mode 100644 index 0000000..7739565 --- /dev/null +++ b/src/main/resources/js/getElementsFromPoint.js @@ -0,0 +1 @@ +return document.elementsFromPoint(arguments[0], arguments[1]); diff --git a/src/test/java/tests/integration/LocatorTests.java b/src/test/java/tests/integration/LocatorTests.java index b2b661c..499ccb0 100644 --- a/src/test/java/tests/integration/LocatorTests.java +++ b/src/test/java/tests/integration/LocatorTests.java @@ -1,16 +1,22 @@ package tests.integration; +import aquality.selenium.browser.AqualityServices; +import aquality.selenium.elements.interfaces.ByImage; import aquality.selenium.elements.interfaces.ILabel; import automationpractice.forms.ChallengingDomForm; import org.openqa.selenium.By; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.locators.RelativeLocator; +import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.testng.asserts.SoftAssert; import tests.BaseTest; import theinternet.TheInternetPage; +import theinternet.forms.BrokenImagesForm; + import java.util.List; + import static aquality.selenium.locators.RelativeBySupplier.with; public class LocatorTests extends BaseTest { @@ -26,6 +32,22 @@ public void beforeMethod() { navigate(TheInternetPage.CHALLENGING_DOM); } + @Test + public void testByImageLocator() { + BrokenImagesForm form = new BrokenImagesForm(); + Assert.assertFalse(form.getLabelByImage().state().isDisplayed(), "Should be impossible to find element on page by image when it is absent"); + getBrowser().goTo(form.getUrl()); + Assert.assertTrue(form.getLabelByImage().state().isDisplayed(), "Should be possible to find element on page by image"); + + List childLabels = form.getChildLabelsByImage(); + List docLabels = form.getLabelsByImage(); + Assert.assertTrue(docLabels.size() > 1, "List of elements should be possible to find by image"); + Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count"); + + ILabel screen = AqualityServices.getElementFactory().getLabel(new ByImage(AqualityServices.getBrowser().getScreenshot()), "full screen"); + Assert.assertTrue(screen.state().waitForDisplayed(), "Should be possible to find element by full page screenshot"); + } + @Test public void testAboveLocatorWithDifferentAboveParametersType() { ILabel cellInRow5Column5 = challengingDomForm.getCellInRow5Column5(); diff --git a/src/test/java/theinternet/forms/BrokenImagesForm.java b/src/test/java/theinternet/forms/BrokenImagesForm.java new file mode 100644 index 0000000..9cffb24 --- /dev/null +++ b/src/test/java/theinternet/forms/BrokenImagesForm.java @@ -0,0 +1,33 @@ +package theinternet.forms; + +import aquality.selenium.elements.interfaces.ByImage; +import aquality.selenium.elements.interfaces.ILabel; +import org.openqa.selenium.By; +import utils.FileUtil; + +import java.util.List; + +public class BrokenImagesForm extends TheInternetForm { + private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png")); + + public BrokenImagesForm(){ + super(By.id("content"), "Broken Images form"); + } + + public ILabel getLabelByImage(){ + return getElementFactory().getLabel(imageLocator, "broken image"); + } + + public List getLabelsByImage(){ + return getElementFactory().findElements(imageLocator, "broken image", ILabel.class); + } + + public List getChildLabelsByImage(){ + return getFormLabel().findChildElements(imageLocator, "broken image", ILabel.class); + } + + @Override + protected String getUri() { + return "/broken_images"; + } +} diff --git a/src/test/resources/brokenImage.png b/src/test/resources/brokenImage.png new file mode 100644 index 0000000000000000000000000000000000000000..9edd76fe2bbe880b1d4555570b78c4e54a3d7ec5 GIT binary patch literal 765 zcmVPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf0+LBYK~y+TtyE1< z6HydBGYnI)VEHh7#fT9hpb3f4L`{eb64ON8xpO6n?EMX{+_*5}js%5nNDT`&SYtE< zBhmPQU`Px>foPl7*3u6;Go8LS-uF74K$Pf$libdmd*3~C&VAES90Cpik|-PwySM+w zz`y|d`ud>jx|^v$5=~EsF>>iLjs%*ppFD*s^Dm-ECgSMtKaIh`v+((RZWU0)k8qba z9sJC3B(~XKBb{d`l^q0wK{Pf7u(0q7rfEtN0hfr=CSb?nD3}~|x`Lh#4aZt)kj`6ZbXr6iIuObExURg7!AIv|w!P=3XLUjSi32)o@T^7ESUZ*Fpl#O^K0-5qAUMBB?H!ON2sZq`M=a7Er5iUr+Q`J5ckhYk1*U4o&XVnJV900000NkvXXu0mjfaLiF2 literal 0 HcmV?d00001 From f6a94f0d83280a4e73df79fbb1aed07d6985dfc6 Mon Sep 17 00:00:00 2001 From: Aleksey2 Meleshko Date: Wed, 26 Apr 2023 17:39:18 +0200 Subject: [PATCH 4/9] stabilize many tools tests --- src/test/java/manytools/ManyToolsForm.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/java/manytools/ManyToolsForm.java b/src/test/java/manytools/ManyToolsForm.java index 5eda112..2849ccf 100644 --- a/src/test/java/manytools/ManyToolsForm.java +++ b/src/test/java/manytools/ManyToolsForm.java @@ -1,9 +1,13 @@ package manytools; import aquality.selenium.browser.AqualityServices; +import aquality.selenium.core.utilities.IActionRetrier; import aquality.selenium.elements.interfaces.ILabel; import aquality.selenium.forms.Form; import org.openqa.selenium.By; +import org.openqa.selenium.TimeoutException; + +import java.util.Collections; public abstract class ManyToolsForm> extends Form { private static final String BASE_URL = "https://manytools.org/"; @@ -17,8 +21,10 @@ protected ManyToolsForm(String name) { @SuppressWarnings("unchecked") public T open() { - AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart()); - AqualityServices.getBrowser().waitForPageToLoad(); + AqualityServices.get(IActionRetrier.class).doWithRetry(() -> { + AqualityServices.getBrowser().goTo(BASE_URL + getUrlPart()); + AqualityServices.getBrowser().waitForPageToLoad(); + }, Collections.singletonList(TimeoutException.class)); return (T) this; } From ade1afe92ff14f2c4e7aa0efc67036b9fea02c28 Mon Sep 17 00:00:00 2001 From: Aleksey2 Meleshko Date: Wed, 26 Apr 2023 22:52:14 +0200 Subject: [PATCH 5/9] Implement screenshot scaling in case when devicePixelRatio!=1 (like on modern Mac) --- .../aquality/selenium/browser/JavaScript.java | 1 + .../selenium/elements/interfaces/ByImage.java | 29 +++++++++++++++++++ src/main/resources/js/getDevicePixelRatio.js | 1 + .../theinternet/forms/BrokenImagesForm.java | 2 +- 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/js/getDevicePixelRatio.js diff --git a/src/main/java/aquality/selenium/browser/JavaScript.java b/src/main/java/aquality/selenium/browser/JavaScript.java index f8f13d0..04e5a13 100644 --- a/src/main/java/aquality/selenium/browser/JavaScript.java +++ b/src/main/java/aquality/selenium/browser/JavaScript.java @@ -21,6 +21,7 @@ public enum JavaScript { GET_CHECKBOX_STATE("getCheckBxState.js"), GET_COMBOBOX_SELECTED_TEXT("getCmbText.js"), GET_COMBOBOX_TEXTS("getCmbValues.js"), + GET_DEVICE_PIXEL_RATIO("getDevicePixelRatio.js"), GET_ELEMENT_BY_XPATH("getElementByXpath.js"), GET_ELEMENTS_FROM_POINT("getElementsFromPoint.js"), GET_ELEMENT_XPATH("getElementXPath.js"), diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java index b7ccc8f..96bc864 100644 --- a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -23,6 +23,7 @@ public class ByImage extends By { private static boolean wasLibraryLoaded = false; private final Mat template; + private final boolean doScaling; private static void loadLibrary() { if (!wasLibraryLoaded) { @@ -38,8 +39,19 @@ private static void loadLibrary() { * @param file image file to locate element by. */ public ByImage(File file) { + this(file, false); + } + + /** + * Constructor accepting image file. + * + * @param file image file to locate element by. + * @param doScaling perform screenshot scaling if devicePixelRatio != 1 + */ + public ByImage(File file, boolean doScaling) { loadLibrary(); this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED); + this.doScaling = doScaling; } /** @@ -48,8 +60,19 @@ public ByImage(File file) { * @param bytes image bytes to locate element by. */ public ByImage(byte[] bytes) { + this(bytes, false); + } + + /** + * Constructor accepting image file. + * + * @param bytes image bytes to locate element by. + * @param doScaling perform screenshot scaling if devicePixelRatio != 1 + */ + public ByImage(byte[] bytes, boolean doScaling) { loadLibrary(); this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); + this.doScaling = doScaling; } @Override @@ -61,6 +84,12 @@ public String toString() { public List findElements(SearchContext context) { byte[] screenshotBytes = getScreenshot(context); Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED); + long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO); + if (devicePixelRatio != 1 && doScaling) { + int scaledWidth = (int) (source.width() / devicePixelRatio); + int scaledHeight = (int) (source.height() / devicePixelRatio); + Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA); + } Mat result = new Mat(); Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED); diff --git a/src/main/resources/js/getDevicePixelRatio.js b/src/main/resources/js/getDevicePixelRatio.js new file mode 100644 index 0000000..679d44c --- /dev/null +++ b/src/main/resources/js/getDevicePixelRatio.js @@ -0,0 +1 @@ +return window.devicePixelRatio; diff --git a/src/test/java/theinternet/forms/BrokenImagesForm.java b/src/test/java/theinternet/forms/BrokenImagesForm.java index 9cffb24..5069876 100644 --- a/src/test/java/theinternet/forms/BrokenImagesForm.java +++ b/src/test/java/theinternet/forms/BrokenImagesForm.java @@ -8,7 +8,7 @@ import java.util.List; public class BrokenImagesForm extends TheInternetForm { - private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png")); + private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png"), true); public BrokenImagesForm(){ super(By.id("content"), "Broken Images form"); From f3817db5a3a0a69617f06c997cc9f59b27b511fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alaksiej=20Miale=C5=A1ka?= Date: Thu, 27 Apr 2023 15:48:31 +0200 Subject: [PATCH 6/9] Make scaling obligatory for full screenshot, rework getScreenshot method, remove from constructor parameters. Add more test actions --- .../selenium/elements/interfaces/ByImage.java | 54 ++++++------------- .../java/tests/integration/LocatorTests.java | 9 +++- .../theinternet/forms/BrokenImagesForm.java | 2 +- 3 files changed, 25 insertions(+), 40 deletions(-) diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java index 96bc864..97b8569 100644 --- a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -18,12 +18,12 @@ /** * Locator to search elements by image. * Takes screenshot and finds match using openCV. + * Performs screenshot scaling if devicePixelRatio != 1. * Then finds elements by coordinates using javascript. */ public class ByImage extends By { private static boolean wasLibraryLoaded = false; private final Mat template; - private final boolean doScaling; private static void loadLibrary() { if (!wasLibraryLoaded) { @@ -39,19 +39,8 @@ private static void loadLibrary() { * @param file image file to locate element by. */ public ByImage(File file) { - this(file, false); - } - - /** - * Constructor accepting image file. - * - * @param file image file to locate element by. - * @param doScaling perform screenshot scaling if devicePixelRatio != 1 - */ - public ByImage(File file, boolean doScaling) { loadLibrary(); this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED); - this.doScaling = doScaling; } /** @@ -60,19 +49,8 @@ public ByImage(File file, boolean doScaling) { * @param bytes image bytes to locate element by. */ public ByImage(byte[] bytes) { - this(bytes, false); - } - - /** - * Constructor accepting image file. - * - * @param bytes image bytes to locate element by. - * @param doScaling perform screenshot scaling if devicePixelRatio != 1 - */ - public ByImage(byte[] bytes, boolean doScaling) { loadLibrary(); this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); - this.doScaling = doScaling; } @Override @@ -82,21 +60,14 @@ public String toString() { @Override public List findElements(SearchContext context) { - byte[] screenshotBytes = getScreenshot(context); - Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED); - long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO); - if (devicePixelRatio != 1 && doScaling) { - int scaledWidth = (int) (source.width() / devicePixelRatio); - int scaledHeight = (int) (source.height() / devicePixelRatio); - Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA); - } + Mat source = getScreenshot(context); Mat result = new Mat(); Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED); float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold(); Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result); - int matchCounter = (result.width() - template.width() + 1) * (result.height() - template.height() + 1); + int matchCounter = Math.abs((result.width() - template.width() + 1) * (result.height() - template.height() + 1)); List matchLocations = new ArrayList<>(); while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) { matchCounter--; @@ -149,11 +120,20 @@ protected static double distanceToPoint(Point matchLocation, WebElement element) * Takes screenshot from searchContext if supported, or from browser. * * @param context search context for element location. - * @return captured screenshot as byte array. + * @return captured screenshot as Mat object. */ - protected byte[] getScreenshot(SearchContext context) { - return !(context instanceof TakesScreenshot) - ? AqualityServices.getBrowser().getScreenshot() - : ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES); + protected Mat getScreenshot(SearchContext context) { + byte[] screenshotBytes = context instanceof TakesScreenshot + ? ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES) + : AqualityServices.getBrowser().getScreenshot(); + boolean isBrowserScreenshot = context instanceof WebDriver || !(context instanceof TakesScreenshot); + Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED); + long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO); + if (devicePixelRatio != 1 && isBrowserScreenshot) { + int scaledWidth = (int) (source.width() / devicePixelRatio); + int scaledHeight = (int) (source.height() / devicePixelRatio); + Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA); + } + return source; } } diff --git a/src/test/java/tests/integration/LocatorTests.java b/src/test/java/tests/integration/LocatorTests.java index 499ccb0..424018b 100644 --- a/src/test/java/tests/integration/LocatorTests.java +++ b/src/test/java/tests/integration/LocatorTests.java @@ -5,6 +5,7 @@ import aquality.selenium.elements.interfaces.ILabel; import automationpractice.forms.ChallengingDomForm; import org.openqa.selenium.By; +import org.openqa.selenium.OutputType; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.locators.RelativeLocator; import org.testng.Assert; @@ -38,14 +39,18 @@ public void testByImageLocator() { Assert.assertFalse(form.getLabelByImage().state().isDisplayed(), "Should be impossible to find element on page by image when it is absent"); getBrowser().goTo(form.getUrl()); Assert.assertTrue(form.getLabelByImage().state().isDisplayed(), "Should be possible to find element on page by image"); + Assert.assertEquals(form.getLabelByImage().getElement().getTagName(), "img", "Correct element must be found"); List childLabels = form.getChildLabelsByImage(); List docLabels = form.getLabelsByImage(); Assert.assertTrue(docLabels.size() > 1, "List of elements should be possible to find by image"); Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count"); - ILabel screen = AqualityServices.getElementFactory().getLabel(new ByImage(AqualityServices.getBrowser().getScreenshot()), "full screen"); - Assert.assertTrue(screen.state().waitForDisplayed(), "Should be possible to find element by full page screenshot"); + ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag"); + ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)), + "full screen"); + Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot"); + Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found"); } @Test diff --git a/src/test/java/theinternet/forms/BrokenImagesForm.java b/src/test/java/theinternet/forms/BrokenImagesForm.java index 5069876..9cffb24 100644 --- a/src/test/java/theinternet/forms/BrokenImagesForm.java +++ b/src/test/java/theinternet/forms/BrokenImagesForm.java @@ -8,7 +8,7 @@ import java.util.List; public class BrokenImagesForm extends TheInternetForm { - private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png"), true); + private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png")); public BrokenImagesForm(){ super(By.id("content"), "Broken Images form"); From 607c032a0ab3f2162a3212114c9f05f4c473707d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alaksiej=20Miale=C5=A1ka?= Date: Thu, 27 Apr 2023 16:04:20 +0200 Subject: [PATCH 7/9] Make scaling obligatory for full screenshot, rework getScreenshot method, remove from constructor parameters. Add more test actions (#124) --- .../selenium/elements/interfaces/ByImage.java | 54 ++++++------------- .../java/tests/integration/LocatorTests.java | 9 +++- .../theinternet/forms/BrokenImagesForm.java | 2 +- 3 files changed, 25 insertions(+), 40 deletions(-) diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java index 96bc864..97b8569 100644 --- a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -18,12 +18,12 @@ /** * Locator to search elements by image. * Takes screenshot and finds match using openCV. + * Performs screenshot scaling if devicePixelRatio != 1. * Then finds elements by coordinates using javascript. */ public class ByImage extends By { private static boolean wasLibraryLoaded = false; private final Mat template; - private final boolean doScaling; private static void loadLibrary() { if (!wasLibraryLoaded) { @@ -39,19 +39,8 @@ private static void loadLibrary() { * @param file image file to locate element by. */ public ByImage(File file) { - this(file, false); - } - - /** - * Constructor accepting image file. - * - * @param file image file to locate element by. - * @param doScaling perform screenshot scaling if devicePixelRatio != 1 - */ - public ByImage(File file, boolean doScaling) { loadLibrary(); this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED); - this.doScaling = doScaling; } /** @@ -60,19 +49,8 @@ public ByImage(File file, boolean doScaling) { * @param bytes image bytes to locate element by. */ public ByImage(byte[] bytes) { - this(bytes, false); - } - - /** - * Constructor accepting image file. - * - * @param bytes image bytes to locate element by. - * @param doScaling perform screenshot scaling if devicePixelRatio != 1 - */ - public ByImage(byte[] bytes, boolean doScaling) { loadLibrary(); this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); - this.doScaling = doScaling; } @Override @@ -82,21 +60,14 @@ public String toString() { @Override public List findElements(SearchContext context) { - byte[] screenshotBytes = getScreenshot(context); - Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED); - long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO); - if (devicePixelRatio != 1 && doScaling) { - int scaledWidth = (int) (source.width() / devicePixelRatio); - int scaledHeight = (int) (source.height() / devicePixelRatio); - Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA); - } + Mat source = getScreenshot(context); Mat result = new Mat(); Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED); float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold(); Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result); - int matchCounter = (result.width() - template.width() + 1) * (result.height() - template.height() + 1); + int matchCounter = Math.abs((result.width() - template.width() + 1) * (result.height() - template.height() + 1)); List matchLocations = new ArrayList<>(); while (matchCounter > 0 && minMaxLoc.maxVal >= threshold) { matchCounter--; @@ -149,11 +120,20 @@ protected static double distanceToPoint(Point matchLocation, WebElement element) * Takes screenshot from searchContext if supported, or from browser. * * @param context search context for element location. - * @return captured screenshot as byte array. + * @return captured screenshot as Mat object. */ - protected byte[] getScreenshot(SearchContext context) { - return !(context instanceof TakesScreenshot) - ? AqualityServices.getBrowser().getScreenshot() - : ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES); + protected Mat getScreenshot(SearchContext context) { + byte[] screenshotBytes = context instanceof TakesScreenshot + ? ((TakesScreenshot) context).getScreenshotAs(OutputType.BYTES) + : AqualityServices.getBrowser().getScreenshot(); + boolean isBrowserScreenshot = context instanceof WebDriver || !(context instanceof TakesScreenshot); + Mat source = Imgcodecs.imdecode(new MatOfByte(screenshotBytes), Imgcodecs.IMREAD_UNCHANGED); + long devicePixelRatio = (long) AqualityServices.getBrowser().executeScript(JavaScript.GET_DEVICE_PIXEL_RATIO); + if (devicePixelRatio != 1 && isBrowserScreenshot) { + int scaledWidth = (int) (source.width() / devicePixelRatio); + int scaledHeight = (int) (source.height() / devicePixelRatio); + Imgproc.resize(source, source, new Size(scaledWidth, scaledHeight), 0, 0, Imgproc.INTER_AREA); + } + return source; } } diff --git a/src/test/java/tests/integration/LocatorTests.java b/src/test/java/tests/integration/LocatorTests.java index 499ccb0..424018b 100644 --- a/src/test/java/tests/integration/LocatorTests.java +++ b/src/test/java/tests/integration/LocatorTests.java @@ -5,6 +5,7 @@ import aquality.selenium.elements.interfaces.ILabel; import automationpractice.forms.ChallengingDomForm; import org.openqa.selenium.By; +import org.openqa.selenium.OutputType; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.locators.RelativeLocator; import org.testng.Assert; @@ -38,14 +39,18 @@ public void testByImageLocator() { Assert.assertFalse(form.getLabelByImage().state().isDisplayed(), "Should be impossible to find element on page by image when it is absent"); getBrowser().goTo(form.getUrl()); Assert.assertTrue(form.getLabelByImage().state().isDisplayed(), "Should be possible to find element on page by image"); + Assert.assertEquals(form.getLabelByImage().getElement().getTagName(), "img", "Correct element must be found"); List childLabels = form.getChildLabelsByImage(); List docLabels = form.getLabelsByImage(); Assert.assertTrue(docLabels.size() > 1, "List of elements should be possible to find by image"); Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count"); - ILabel screen = AqualityServices.getElementFactory().getLabel(new ByImage(AqualityServices.getBrowser().getScreenshot()), "full screen"); - Assert.assertTrue(screen.state().waitForDisplayed(), "Should be possible to find element by full page screenshot"); + ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag"); + ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)), + "full screen"); + Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot"); + Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found"); } @Test diff --git a/src/test/java/theinternet/forms/BrokenImagesForm.java b/src/test/java/theinternet/forms/BrokenImagesForm.java index 5069876..9cffb24 100644 --- a/src/test/java/theinternet/forms/BrokenImagesForm.java +++ b/src/test/java/theinternet/forms/BrokenImagesForm.java @@ -8,7 +8,7 @@ import java.util.List; public class BrokenImagesForm extends TheInternetForm { - private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png"), true); + private final By imageLocator = new ByImage(FileUtil.getResourceFileByName("brokenImage.png")); public BrokenImagesForm(){ super(By.id("content"), "Broken Images form"); From 108f28839e2d55883d54812061c52ba4da4be751 Mon Sep 17 00:00:00 2001 From: Aleksey2 Meleshko Date: Thu, 27 Apr 2023 22:28:06 +0200 Subject: [PATCH 8/9] Implement equals and toString properly, fix javadoc --- .../selenium/elements/interfaces/ByImage.java | 24 +++++++++++++++++-- .../java/tests/integration/LocatorTests.java | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java index 97b8569..518f544 100644 --- a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -24,6 +24,7 @@ public class ByImage extends By { private static boolean wasLibraryLoaded = false; private final Mat template; + private final String description; private static void loadLibrary() { if (!wasLibraryLoaded) { @@ -40,24 +41,43 @@ private static void loadLibrary() { */ public ByImage(File file) { loadLibrary(); + description = file.getName(); this.template = Imgcodecs.imread(file.getAbsolutePath(), Imgcodecs.IMREAD_UNCHANGED); } /** - * Constructor accepting image file. + * Constructor accepting image bytes. * * @param bytes image bytes to locate element by. */ public ByImage(byte[] bytes) { loadLibrary(); + description = String.format("bytes[%d]", bytes.length); this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); } @Override public String toString() { - return "ByImage: " + new Dimension(template.width(), template.height()); + return String.format("ByImage: %s, size: (width:%d, height:%d)", description, template.width(), template.height()); } + @Override + public boolean equals(Object o) { + if (!(o instanceof ByImage)) { + return false; + } + + ByImage that = (ByImage) o; + + return this.template.equals(that.template); + } + + @Override + public int hashCode() { + return template.hashCode(); + } + + @Override public List findElements(SearchContext context) { Mat source = getScreenshot(context); diff --git a/src/test/java/tests/integration/LocatorTests.java b/src/test/java/tests/integration/LocatorTests.java index 424018b..adab3b2 100644 --- a/src/test/java/tests/integration/LocatorTests.java +++ b/src/test/java/tests/integration/LocatorTests.java @@ -48,7 +48,7 @@ public void testByImageLocator() { ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag"); ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)), - "full screen"); + "body screen"); Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot"); Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found"); } From 21c170b6e8cbafbd4c5a7c4f60ac0a774c26a60e Mon Sep 17 00:00:00 2001 From: Aleksey2 Meleshko Date: Mon, 8 May 2023 21:14:34 +0200 Subject: [PATCH 9/9] Add possibility to change the threshold for one ByImage locator, with border value check --- .../selenium/elements/interfaces/ByImage.java | 23 ++++++++++++++++++- .../java/tests/integration/LocatorTests.java | 4 +++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java index 518f544..7a8af48 100644 --- a/src/main/java/aquality/selenium/elements/interfaces/ByImage.java +++ b/src/main/java/aquality/selenium/elements/interfaces/ByImage.java @@ -25,6 +25,7 @@ public class ByImage extends By { private static boolean wasLibraryLoaded = false; private final Mat template; private final String description; + private float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold(); private static void loadLibrary() { if (!wasLibraryLoaded) { @@ -56,6 +57,27 @@ public ByImage(byte[] bytes) { this.template = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_UNCHANGED); } + /** + * Sets threshold of image similarity. + * @param threshold a float between 0 and 1, where 1 means 100% match, and 0.5 means 50% match. + * @return current instance of ByImage locator. + */ + public ByImage setThreshold(float threshold) { + if (threshold < 0 || threshold > 1) { + throw new IllegalArgumentException("Threshold must be a float between 0 and 1."); + } + this.threshold = threshold; + return this; + } + + /** + * Gets threshold of image similarity. + * @return current value of threshold. + */ + public float getThreshold() { + return threshold; + } + @Override public String toString() { return String.format("ByImage: %s, size: (width:%d, height:%d)", description, template.width(), template.height()); @@ -84,7 +106,6 @@ public List findElements(SearchContext context) { Mat result = new Mat(); Imgproc.matchTemplate(source, template, result, Imgproc.TM_CCOEFF_NORMED); - float threshold = 1 - AqualityServices.getConfiguration().getVisualizationConfiguration().getDefaultThreshold(); Core.MinMaxLocResult minMaxLoc = Core.minMaxLoc(result); int matchCounter = Math.abs((result.width() - template.width() + 1) * (result.height() - template.height() + 1)); diff --git a/src/test/java/tests/integration/LocatorTests.java b/src/test/java/tests/integration/LocatorTests.java index adab3b2..43e92e6 100644 --- a/src/test/java/tests/integration/LocatorTests.java +++ b/src/test/java/tests/integration/LocatorTests.java @@ -47,9 +47,11 @@ public void testByImageLocator() { Assert.assertEquals(docLabels.size(), childLabels.size(), "Should be possible to find child elements by image with the same count"); ILabel documentByTag = AqualityServices.getElementFactory().getLabel(By.tagName("body"), "document by tag"); - ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)), + float fullThreshold = 1; + ILabel documentByImage = AqualityServices.getElementFactory().getLabel(new ByImage(documentByTag.getElement().getScreenshotAs(OutputType.BYTES)).setThreshold(fullThreshold), "body screen"); Assert.assertTrue(documentByImage.state().isDisplayed(), "Should be possible to find element by document screenshot"); + Assert.assertEquals(((ByImage)documentByImage.getLocator()).getThreshold(), fullThreshold, "Should be possible to get ByImage threshold"); Assert.assertEquals(documentByImage.getElement().getTagName(), "body", "Correct element must be found"); }