From 5e9eca59875efca2ff2324ee99197b741a30c26d Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 21 Nov 2020 21:30:26 +0100 Subject: [PATCH 01/40] Added base class for extension settings. The current key value solution is not very scalable and is becoming a monolith. The extended setting class uses XML and makes it possible to create new "setting groups" independent from each other. --- testar/src/org/fruit/monkey/ConfigTags.java | 1 + testar/src/org/fruit/monkey/Settings.java | 7 + .../src/org/fruit/monkey/SettingsDialog.java | 8 + .../org/testar/settings/ExtendedSettings.java | 271 ++++++++++++++++++ .../testar/settings/ExtendedSettingsTest.java | 238 +++++++++++++++ testar/test/org/testar/settings/XmlFile.java | 2 +- 6 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 testar/src/org/testar/settings/ExtendedSettings.java create mode 100644 testar/test/org/testar/settings/ExtendedSettingsTest.java diff --git a/testar/src/org/fruit/monkey/ConfigTags.java b/testar/src/org/fruit/monkey/ConfigTags.java index fa53fe49b..8af983749 100644 --- a/testar/src/org/fruit/monkey/ConfigTags.java +++ b/testar/src/org/fruit/monkey/ConfigTags.java @@ -34,6 +34,7 @@ import org.fruit.Pair; import org.fruit.alayer.Tag; +import java.nio.file.Path; import java.util.List; public final class ConfigTags { diff --git a/testar/src/org/fruit/monkey/Settings.java b/testar/src/org/fruit/monkey/Settings.java index 440ac29f1..3924c6403 100644 --- a/testar/src/org/fruit/monkey/Settings.java +++ b/testar/src/org/fruit/monkey/Settings.java @@ -443,6 +443,13 @@ public String toFileString() throws IOException{ +"ExtendedSettingsFile =" + Util.lineSep() +"\n" +"#################################################################\n" + +"# Extended settings file\n" + +"#\n" + +"# Relative path to extended settings file.\n" + +"#################################################################\n" + +"ExtendedSettingsFile =" + Util.lineSep() + +"\n" + +"#################################################################\n" +"# Other more advanced settings\n" +"#################################################################\n"); diff --git a/testar/src/org/fruit/monkey/SettingsDialog.java b/testar/src/org/fruit/monkey/SettingsDialog.java index 3001ca9ca..a5392c670 100644 --- a/testar/src/org/fruit/monkey/SettingsDialog.java +++ b/testar/src/org/fruit/monkey/SettingsDialog.java @@ -36,6 +36,7 @@ import org.fruit.Pair; import org.fruit.Util; +import org.fruit.alayer.exceptions.NoSuchTagException; import org.fruit.monkey.dialog.*; import org.testar.settings.ExtendedSettingsFactory; @@ -71,6 +72,7 @@ public class SettingsDialog extends JFrame implements Observer { private static final long serialVersionUID = 5156320008281200950L; static final String TESTAR_VERSION = "2.3.4 (1-Jul-2021)"; + static final String SETTINGS_FILENAME = "test.settings"; private String settingsFile; private Settings settings; @@ -185,6 +187,12 @@ private void checkSettings(Settings settings) throws IllegalStateException { throw new IllegalStateException("Temp Directory does not exist!"); } + try{ + settings.get(ConfigTags.ExtendedSettingsFile); + } catch (NoSuchTagException e){ + settings.set(ConfigTags.ExtendedSettingsFile, settingsFile.replace(SETTINGS_FILENAME, ExtendedSettings.FileName)); + } + settingPanels.forEach((k,v) -> v.right().checkSettings()); } diff --git a/testar/src/org/testar/settings/ExtendedSettings.java b/testar/src/org/testar/settings/ExtendedSettings.java new file mode 100644 index 000000000..506e454eb --- /dev/null +++ b/testar/src/org/testar/settings/ExtendedSettings.java @@ -0,0 +1,271 @@ +package org.testar.settings; + +import org.apache.commons.lang.ObjectUtils; +import org.apache.commons.lang.SerializationUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.checkerframework.checker.nullness.qual.NonNull; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAnyElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Generic XML root element. + */ +@XmlRootElement(name = "root") +@XmlAccessorType(XmlAccessType.FIELD) +class ExtendedSettingsXml implements Serializable { + // Holds all the XML elements found in the file. + @XmlAnyElement(lax = true) + public List any; + + public Integer version; + + public ExtendedSettingsXml() { + any = new ArrayList<>(); + version = 1; + } +} + +/** + * Helper class for reading XML data from disk. + */ +class ExtractionResult { + final JAXBContext Context; + final ExtendedSettingsXml Data; + final Boolean FileNotFound; + + public ExtractionResult(JAXBContext context, ExtendedSettingsXml data, Boolean fileNotFound) { + Context = context; + Data = data; + FileNotFound = fileNotFound; + } +} + +public class ExtendedSettings implements Serializable { + private static final Logger LOGGER = LogManager.getLogger(); + public static final String FileName = "ExtendedSettings.xml"; + private final String _absolutePath; + private final ReentrantReadWriteLock _fileAccessMutex; + + /** + * Stores a deep copy of the object up on reading. The {@link #load(Class)} returns a reference to the XML element. + * If we modify that reference and want to save it, we need to replace the old value. We use {@code #_loadedValue} + * to find and update the tag in the XML file. + */ + private Object _loadedValue = null; + + /** + * Constructor, each specialization must have a unique implementation of this class. + * + * @param fileLocation The absolute path the the XML file. + * @param fileAccessMutex Mutex for thread-safe access. + */ + protected ExtendedSettings(@NonNull String fileLocation, @NonNull ReentrantReadWriteLock fileAccessMutex) { + _fileAccessMutex = fileAccessMutex; + _absolutePath = System.getProperty("user.dir") + + (fileLocation.startsWith(".") ? fileLocation.substring(1) : (fileLocation.startsWith(File.separator) + ? fileLocation : File.separator + fileLocation)); + } + + /** + * Try to load the requested data element from the XML file. + * + * @param clazz The class type of the element we want to load. + * @param The class type of the element we want to load. + * @return When found in the XML the requested element, otherwise null. + */ + @SuppressWarnings("unchecked") + public T load(@SuppressWarnings("rawtypes") @NonNull Class clazz) { + T result = null; + + // Check if file exits + ExtractionResult rd = readFile(clazz); + + if (Objects.nonNull(rd.Data)) { + // Try to find the section of interest. + result = (T) rd.Data.any.stream() + .filter(element -> element.getClass() == clazz) + .findFirst() + .orElse(null); + + // We only support loading a single element for now. + if (rd.Data.any.stream().filter(element -> element.getClass() == clazz).count() > 1) { + LOGGER.error("Duplicate elements found for {}, returning first element ", clazz); + } + + // Store the current content, so we can replace it when needed. + _loadedValue = SerializationUtils.clone((Serializable) result); + } + if (result == null) { + LOGGER.info("Did not found XML element for class: {}", clazz); + } + + return result; + } + + /** + * Try to load the requested data element from the XML file. + * If not found, the default configuration is written to disk and returned. + * + * @param clazz The class type of the element we want to load. + * @param defaultFunctor The function to create the default configuration for class #clazz. + * @param The class type of the element we want to load. + * @return Either the element found in the file otherwise the default configuration. + */ + public T load(@SuppressWarnings("rawtypes") @NonNull Class clazz, @NonNull IExtendedSettingDefaultValue defaultFunctor) { + T result = load(clazz); + + if (result == null) { + LOGGER.info("Writing default values for {}", clazz); + save(defaultFunctor.CreateDefault()); + return load(clazz); + } + + return result; + } + + /** + * Save the data to the XML file. + * The file is created if it does not exist. + * + * @param data The data we need to store. + */ + public void save(@NonNull Object data) { + if (!(data instanceof Comparable)) { + LOGGER.error("Object {} is not extending Comparable", data); + return; + } + + ExtractionResult result = readFile(data.getClass()); + updateFile(data, result.Context, result.Data); + } + + @SuppressWarnings("rawtypes") + private ExtractionResult extractContent(@NonNull Class clazz) { + JAXBContext context = null; + ExtendedSettingsXml data = null; + boolean fileNotFound = false; + + try { + FileInputStream xmlFile = null; + context = JAXBContext.newInstance(ExtendedSettingsXml.class, clazz); + Unmarshaller um = context.createUnmarshaller(); + + try { + _fileAccessMutex.readLock().lock(); + // Load latest version of XML, other settings may have been updated in the mean time. + xmlFile = new FileInputStream(_absolutePath); + data = (ExtendedSettingsXml) um.unmarshal(xmlFile); + } catch (FileNotFoundException e) { + fileNotFound = true; + } finally { + if (Objects.nonNull(xmlFile)) { + xmlFile.close(); + } + } + } catch (IOException | JAXBException e) { + e.printStackTrace(); + } finally { + _fileAccessMutex.readLock().unlock(); + } + return new ExtractionResult(context, data, fileNotFound); + } + + @SuppressWarnings("rawtypes") + private ExtractionResult readFile(@NonNull Class clazz) { + + ExtractionResult result = extractContent(clazz); + if (result.FileNotFound) { + createExtendedSettingsFile(); + result = extractContent(clazz); + } + + return result; + } + + @SuppressWarnings("rawtypes") + private void updateFile(@NonNull Object data, JAXBContext context, ExtendedSettingsXml xmlData) { + Objects.requireNonNull(context); + Objects.requireNonNull(xmlData); + + try { + _fileAccessMutex.writeLock().lock(); + OutputStream os = null; + try { + os = new FileOutputStream(_absolutePath); + Marshaller marshaller = context.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + + // Try to find the old XML tag. + Object found = null; + for (Object element : xmlData.any) { + if (element.getClass() == data.getClass()) { + if (ObjectUtils.compare((Comparable) element, (Comparable) _loadedValue) == 0) { + found = element; + } + } + } + + // Replace old content with new. + xmlData.any.remove(found); + xmlData.any.add(data); + + // Update the file. + marshaller.marshal(xmlData, os); + } catch (JAXBException e) { + e.printStackTrace(); + } finally { + if (Objects.nonNull(os)) { + os.close(); + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + _fileAccessMutex.writeLock().unlock(); + } + } + + private void createExtendedSettingsFile() { + try { + _fileAccessMutex.writeLock().lock(); + OutputStream os = null; + try { + os = new FileOutputStream(_absolutePath); + JAXBContext context = JAXBContext.newInstance(ExtendedSettingsXml.class); + Marshaller marshaller = context.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + marshaller.marshal(new ExtendedSettingsXml(), os); + LOGGER.info("Created extended settings file: {}", _absolutePath); + } catch (JAXBException e) { + e.printStackTrace(); + } finally { + if (Objects.nonNull(os)) { + os.close(); + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + _fileAccessMutex.writeLock().unlock(); + } + } +} diff --git a/testar/test/org/testar/settings/ExtendedSettingsTest.java b/testar/test/org/testar/settings/ExtendedSettingsTest.java new file mode 100644 index 000000000..c610b66da --- /dev/null +++ b/testar/test/org/testar/settings/ExtendedSettingsTest.java @@ -0,0 +1,238 @@ +package org.testar.settings; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.Arrays; +import java.util.Objects; +import java.util.Scanner; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.junit.Assert.*; + +public class ExtendedSettingsTest { + final private String _rootDir = System.getProperty("user.dir") + File.separator; + final private String _relativePath = "extended_settings_test" + File.separatorChar; + final private String _workingDir = _rootDir + _relativePath; + + ExtendedSettings sut; + ReentrantReadWriteLock fileAccessLock; + + @After + public void CleanUp() { + removeAllXmlTestFiles(); + } + + @Before + public void Setup() { + File directory = new File(_workingDir); + if (!directory.exists()) { + assertTrue(directory.mkdir()); + } + removeAllXmlTestFiles(); + + fileAccessLock = new ReentrantReadWriteLock(); + } + + void removeAllXmlTestFiles() { + // Search and delete all XML files + for (File file : Objects.requireNonNull(new File(_workingDir).listFiles())) { + if (file.getName().endsWith(".xml")) { + assertTrue(file.delete()); + } + } + } + + Boolean fileContains(String absolutePath, String... expectedLines) { + Boolean[] result = new Boolean[expectedLines.length]; + Arrays.fill(result, Boolean.FALSE); + + try { + File file = new File(absolutePath); + for (int i = 0; i < expectedLines.length; i++) { + Scanner myReader = new Scanner(file); + while (myReader.hasNextLine()) { + String data = myReader.nextLine(); + if (data.replaceAll("\\s+", "") + .equalsIgnoreCase(expectedLines[i].replaceAll("\\s+", ""))) { + result[i] = true; + break; + } + } + myReader.close(); + } + } catch (FileNotFoundException e) { + System.out.println("An error occurred."); + e.printStackTrace(); + } + + return Arrays.stream(result).allMatch(x -> x); + } + + @Test + public void settingFileCreationWhenNotExisting() { + // GIVEN The file doesn't exist. + final String unknownFile = "ghost.xml"; + File testFile = new File(_workingDir + unknownFile); + assertFalse(testFile.exists()); + + // WHEN Trying to load an extended setting without default values. + sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); + TestSetting element = sut.load(TestSetting.class); + + // THEN The file is created. + assertTrue(testFile.exists()); + assertNull(element); + } + + @Test + public void settingFileCreationWithDefaultValuesWhenNotExisting() { + // GIVEN The file doesn't exist. + final String unknownFile = "default.xml"; + File testFile = new File(_workingDir + unknownFile); + assertFalse(testFile.exists()); + + // WHEN Trying to load an extended setting without default values. + sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); + TestSetting element = sut.load(TestSetting.class, TestSetting::CreateDefault); + + // THEN The file is created containing default values. + assertTrue(testFile.exists()); + assertNotNull(element); + assertEquals(TestSetting.DEFAULT_VALUE, element.value); + } + + @Test() + public void saveValueWhenFileNotExists() { + // GIVEN The file doesn't exist. + final String unknownFile = "save.xml"; + File testFile = new File(_workingDir + unknownFile); + assertFalse(testFile.exists()); + + // WHEN Trying to save an extended setting before loading. + sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); + TestSetting element = new TestSetting(); + sut.save(element); + + // THEN The file is created. + assertTrue(testFile.exists()); + } + + @Test + public void loadFromFileWithOnlyUnknownElements() { + // GIVEN The file contains only unknown elements. + final String unknownFile = "unknown.xml"; + XmlFile.CreateUnknownFile(_workingDir + unknownFile); + + // WHEN Trying to load an extended setting without default values. + sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); + TestSetting element = sut.load(TestSetting.class); + + // THEN The element remains empty. + assertNull(element); + } + + @Test + public void saveToFileWithOnlyUnknownElements() { + // GIVEN The file contains only unknown elements. + final String unknownFile = "unknown_save.xml"; + XmlFile.CreateUnknownFile(_workingDir + unknownFile); + + // WHEN Trying to save an extended setting. + sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); + TestSetting data = TestSetting.CreateDefault(); + sut.save(data); + + // THEN The data is added to the file. + assertTrue(fileContains(_workingDir + unknownFile, "", + "Default", + "")); + } + + @Test + public void updateKnownElement() { + // GIVEN The file contains one known element. + final String knownFile = "update_known.xml"; + XmlFile.CreateSingleTestSetting(_workingDir + knownFile); + + // WHEN Trying to update a known element. + sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); + TestSetting data = sut.load(TestSetting.class); + data.value = "updated"; + sut.save(data); + + // THEN The data is updated and saved to the file. + assertTrue(fileContains(_workingDir + knownFile, "", + "updated", + "")); + } + + @Test + public void updateKnownElementWhenFileMultipleKnownElements() { + // GIVEN The file contains multiple known elements. + final String knownFile = "update_known_multiple.xml"; + XmlFile.CreateMultipleTestSetting(_workingDir + knownFile); + + // WHEN Trying to update a known element. + sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); + TestSetting data = sut.load(TestSetting.class); + data.value = "version3"; + sut.save(data); + + // THEN The first element is updated and saved to the file. + assertTrue(fileContains(_workingDir + knownFile, "version3", "version2")); + assertFalse(fileContains(_workingDir + knownFile, "version1")); + } + + @Test + public void updateMultipleElement() { + // GIVEN The file contains multiple known elements. + final String knownFile = "update_multiple_elements.xml"; + File testFile = new File(_workingDir + knownFile); + assertFalse(testFile.exists()); + sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); + ExtendedSettings sutTwo = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); + TestSetting elementOne = sut.load(TestSetting.class, TestSetting::CreateDefault); + OtherSetting elementTwo = sutTwo.load(OtherSetting.class, OtherSetting::CreateDefault); + assertTrue(testFile.exists()); + assertNotNull(elementOne); + + // WHEN Trying to update the first element. + elementOne.value = "updated"; + sut.save(elementOne); + + // THEN The update is stored. + assertEquals("updated", elementOne.value); + assertTrue(fileContains(_workingDir + knownFile, "updated", "5")); + + // WHEN Trying to update the second element. + elementTwo.speed = 6; + sutTwo.save(elementTwo); + + // THEN The update is stored. + assertEquals(6, elementTwo.speed); + assertTrue(fileContains(_workingDir + knownFile, "updated", "6")); + } + + @Test(expected = ClassCastException.class) + public void updateElementWithWrongBaseClass() { + // GIVEN The file contains multiple known elements. + final String knownFile = "update_wrong_element.xml"; + File testFile = new File(_workingDir + knownFile); + assertFalse(testFile.exists()); + sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); + TestSetting elementOne = sut.load(TestSetting.class, TestSetting::CreateDefault); + OtherSetting elementTwo = sut.load(OtherSetting.class, OtherSetting::CreateDefault); + assertTrue(testFile.exists()); + assertNotNull(elementOne); + + // WHEN Trying to update the first element while the _loadedvalue has been assigned to OtherSetting. + elementOne.value = "updated"; + sut.save(elementOne); + + // THEN An exception should been throw. + } +} diff --git a/testar/test/org/testar/settings/XmlFile.java b/testar/test/org/testar/settings/XmlFile.java index 9944f7cbf..b1b9b46d9 100644 --- a/testar/test/org/testar/settings/XmlFile.java +++ b/testar/test/org/testar/settings/XmlFile.java @@ -99,4 +99,4 @@ private static void CreateFile(final String absolutePath, final String content) File testFile = new File(absolutePath); assertTrue(testFile.exists()); } -} \ No newline at end of file +} From e9b5bc08793f2c5980e84fc1c096d6feaa103981 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 27 Nov 2020 20:20:25 +0100 Subject: [PATCH 02/40] Added initial Tesseract initialization and teardown behavior. --- testar/build.gradle | 1 + .../DummyVisualValidator.java | 15 ++++ .../VisualValidationFactory.java | 19 +++++ .../VisualValidationManager.java | 9 +++ .../VisualValidationSettings.java | 7 +- .../visualvalidation/VisualValidator.java | 39 +++++++++ .../matcher/VisualDummyMatcher.java | 8 ++ .../matcher/VisualMatcher.java | 5 ++ .../matcher/VisualMatcherFactory.java | 5 ++ .../ocr/OcrConfiguration.java | 39 +++++++++ .../ocr/OcrEngineFactory.java | 23 ++++++ .../ocr/OcrEngineInterface.java | 9 +++ .../ocr/dummy/DummyOcrEngine.java | 17 ++++ .../ocr/tesseract/TesseractOcrEngine.java | 80 +++++++++++++++++++ .../ocr/tesseract/TesseractSettings.java | 30 +++++++ .../src/org/fruit/monkey/DefaultProtocol.java | 12 +++ .../testar/settings/ExtendedSettingBase.java | 2 +- .../settings/ExtendedSettingsFactory.java | 7 +- 18 files changed, 324 insertions(+), 3 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/VisualValidator.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/OcrConfiguration.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineFactory.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java diff --git a/testar/build.gradle b/testar/build.gradle index c331fbea1..7bb93d9d7 100644 --- a/testar/build.gradle +++ b/testar/build.gradle @@ -88,6 +88,7 @@ dependencies { compile group: 'org.eclipse.jetty', name: 'jetty-annotations', version: '9.4.30.v20200611' compile group: 'org.eclipse.jetty', name: 'apache-jsp', version: '9.4.30.v20200611' compile group: 'org.eclipse.jetty', name: 'apache-jstl', version: '9.4.30.v20200611' + compile group: 'org.bytedeco', name: 'tesseract-platform', version: '4.1.1-1.5.4' compile group: 'commons-io', name: 'commons-io', version: '2.7' runtime project(':windows') } diff --git a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java new file mode 100644 index 000000000..e695940db --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java @@ -0,0 +1,15 @@ +package nl.ou.testar.visualvalidation; + +import java.awt.image.BufferedImage; + +public class DummyVisualValidator implements VisualValidationManager{ + @Override + public void AnalyzeImage(BufferedImage image) { + + } + + @Override + public void Close() { + + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java new file mode 100644 index 000000000..c15d95283 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java @@ -0,0 +1,19 @@ +package nl.ou.testar.visualvalidation; + +import org.testar.settings.ExtendedSettingsFactory; + +public class VisualValidationFactory { + + public static VisualValidationManager createVisualValidator() { + VisualValidationManager visualValidator; + VisualValidationSettings visualValidation = ExtendedSettingsFactory.createVisualValidationSettings(); + + if (visualValidation.enabled) { + visualValidator = new VisualValidator(visualValidation); + } else { + visualValidator = new DummyVisualValidator(); + } + + return visualValidator; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java new file mode 100644 index 000000000..01273cc6f --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java @@ -0,0 +1,9 @@ +package nl.ou.testar.visualvalidation; + +import java.awt.image.BufferedImage; + +public interface VisualValidationManager { + void AnalyzeImage(BufferedImage image); + + void Close(); +} diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java index f2a57c4e8..692e232d6 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java @@ -30,6 +30,7 @@ package nl.ou.testar.visualvalidation; +import nl.ou.testar.visualvalidation.ocr.OcrConfiguration; import org.testar.settings.ExtendedSettingBase; import javax.xml.bind.annotation.XmlAccessType; @@ -41,10 +42,13 @@ public class VisualValidationSettings extends ExtendedSettingBase { public Boolean enabled; + public OcrConfiguration ocrConfiguration; + @Override public int compareTo(VisualValidationSettings other) { int res = -1; - if (this.enabled.equals(other.enabled)) { + if ((enabled.equals(other.enabled)) && + (ocrConfiguration.compareTo(other.ocrConfiguration) == 0)) { res = 0; } return res; @@ -60,6 +64,7 @@ public String toString() { public static VisualValidationSettings CreateDefault() { VisualValidationSettings DefaultInstance = new VisualValidationSettings(); DefaultInstance.enabled = false; + DefaultInstance.ocrConfiguration = OcrConfiguration.CreateDefault(); return DefaultInstance; } } diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java new file mode 100644 index 000000000..0804c102e --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -0,0 +1,39 @@ +package nl.ou.testar.visualvalidation; + +import nl.ou.testar.visualvalidation.matcher.VisualMatcher; +import nl.ou.testar.visualvalidation.matcher.VisualMatcherFactory; +import nl.ou.testar.visualvalidation.ocr.OcrConfiguration; +import nl.ou.testar.visualvalidation.ocr.OcrEngineFactory; +import nl.ou.testar.visualvalidation.ocr.OcrEngineInterface; + +import java.awt.image.BufferedImage; + +public class VisualValidator implements VisualValidationManager { + private final OcrEngineInterface ocrEngine; + private final VisualMatcher matcher; + + public VisualValidator(VisualValidationSettings settings) { + OcrConfiguration ocrConfig = settings.ocrConfiguration; + if (ocrConfig.enabled) { + ocrEngine = OcrEngineFactory.createOcrEngine(ocrConfig); + } else { + ocrEngine = null; + } + + matcher = VisualMatcherFactory.createDummyMatcher(); + } + + @Override + public void AnalyzeImage(BufferedImage image) { + + ocrEngine.ScanImage(image); + } + + @Override + public void Close() { + matcher.destroy(); + if (ocrEngine != null) { + ocrEngine.Destroy(); + } + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java new file mode 100644 index 000000000..3fa204132 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java @@ -0,0 +1,8 @@ +package nl.ou.testar.visualvalidation.matcher; + +public class VisualDummyMatcher implements VisualMatcher { + @Override + public void destroy() { + + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java new file mode 100644 index 000000000..5291c0380 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java @@ -0,0 +1,5 @@ +package nl.ou.testar.visualvalidation.matcher; + +public interface VisualMatcher { + void destroy(); +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java new file mode 100644 index 000000000..11963b59b --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java @@ -0,0 +1,5 @@ +package nl.ou.testar.visualvalidation.matcher; + +public class VisualMatcherFactory { + public static VisualMatcher createDummyMatcher() { return new VisualDummyMatcher(); } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrConfiguration.java new file mode 100644 index 000000000..4c56904a0 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrConfiguration.java @@ -0,0 +1,39 @@ +package nl.ou.testar.visualvalidation.ocr; + +import org.testar.settings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class OcrConfiguration extends ExtendedSettingBase { + public static final String TESSERACT_ENGINE = "tesseract"; + + public Boolean enabled; + public String engine; + + @Override + public String toString() { + return "OcrConfiguration{" + + "enabled=" + enabled + + ", engine='" + engine + '\'' + + '}'; + } + + public static OcrConfiguration CreateDefault() { + OcrConfiguration instance = new OcrConfiguration(); + instance.enabled = true; + instance.engine = TESSERACT_ENGINE; + return instance; + } + + @Override + public int compareTo(OcrConfiguration other) { + int result = -1; + if ((enabled.equals(other.enabled)) && + (engine.contentEquals(other.engine))) { + result = 0; + } + return result; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineFactory.java b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineFactory.java new file mode 100644 index 000000000..0707d105d --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineFactory.java @@ -0,0 +1,23 @@ +package nl.ou.testar.visualvalidation.ocr; + +import nl.ou.testar.visualvalidation.ocr.dummy.DummyOcrEngine; +import nl.ou.testar.visualvalidation.ocr.tesseract.TesseractOcrEngine; + +public class OcrEngineFactory { + + public static OcrEngineInterface createOcrEngine(OcrConfiguration settings) { + if (settings.engine.contentEquals(OcrConfiguration.TESSERACT_ENGINE)) { + return createTesseractEngine(); + } else { + return createDummyEngine(); + } + } + + static OcrEngineInterface createTesseractEngine() { + return new TesseractOcrEngine(); + } + + static OcrEngineInterface createDummyEngine() { + return new DummyOcrEngine(); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java new file mode 100644 index 000000000..3ce87a655 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java @@ -0,0 +1,9 @@ +package nl.ou.testar.visualvalidation.ocr; + +import java.awt.image.BufferedImage; + +public interface OcrEngineInterface { + void ScanImage(BufferedImage image); + + void Destroy(); +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java new file mode 100644 index 000000000..a335938e5 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java @@ -0,0 +1,17 @@ +package nl.ou.testar.visualvalidation.ocr.dummy; + +import nl.ou.testar.visualvalidation.ocr.OcrEngineInterface; + +import java.awt.image.BufferedImage; + +public class DummyOcrEngine implements OcrEngineInterface { + @Override + public void ScanImage(BufferedImage image) { + + } + + @Override + public void Destroy() { + + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java new file mode 100644 index 000000000..26a4709e3 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java @@ -0,0 +1,80 @@ +package nl.ou.testar.visualvalidation.ocr.tesseract; + +import nl.ou.testar.visualvalidation.ocr.OcrEngineInterface; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.tesseract.TessBaseAPI; +import org.testar.settings.ExtendedSettingsFactory; + +import java.awt.image.BufferedImage; +import java.awt.image.DataBuffer; +import java.awt.image.DataBufferByte; +import java.awt.image.DataBufferInt; +import java.awt.image.DataBufferShort; +import java.awt.image.DataBufferUShort; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; + +public class TesseractOcrEngine implements OcrEngineInterface { + private final TessBaseAPI engine; + static final Logger LOGGER = LogManager.getLogger(); + + public TesseractOcrEngine() { + engine = new TessBaseAPI(); + TesseractSettings config = ExtendedSettingsFactory.createTesseractSetting(); + + if (engine.Init(config.dataPath, config.language) != 0) { + LOGGER.error("Could not initialize tesseract."); + } + + LOGGER.info("Tesseract engine created; Language:{} Data path:{}", config.language, config.dataPath); + } + + @Override + public void ScanImage(BufferedImage image) { + // TODO TM: Image analysis should be done on separate thread. + DataBuffer dataBuffer = image.getData().getDataBuffer(); + + ByteBuffer byteBuffer; + if (dataBuffer instanceof DataBufferByte) { + byte[] pixelData = ((DataBufferByte) dataBuffer).getData(); + byteBuffer = ByteBuffer.wrap(pixelData); + } else if (dataBuffer instanceof DataBufferUShort) { + short[] pixelData = ((DataBufferUShort) dataBuffer).getData(); + byteBuffer = ByteBuffer.allocate(pixelData.length * 2); + byteBuffer.asShortBuffer().put(ShortBuffer.wrap(pixelData)); + } else if (dataBuffer instanceof DataBufferShort) { + short[] pixelData = ((DataBufferShort) dataBuffer).getData(); + byteBuffer = ByteBuffer.allocate(pixelData.length * 2); + byteBuffer.asShortBuffer().put(ShortBuffer.wrap(pixelData)); + } else if (dataBuffer instanceof DataBufferInt) { + int[] pixelData = ((DataBufferInt) dataBuffer).getData(); + byteBuffer = ByteBuffer.allocate(pixelData.length * 4); + byteBuffer.asIntBuffer().put(IntBuffer.wrap(pixelData)); + } else { + throw new IllegalArgumentException("Not implemented for data buffer type: " + dataBuffer.getClass()); + } + + // When applicable, alpha is included. + int bytes_per_pixel = image.getColorModel().getNumComponents(); + int bytes_per_line = bytes_per_pixel * image.getWidth(); + + engine.SetImage(byteBuffer, image.getWidth(), image.getHeight(), bytes_per_pixel, bytes_per_line); + BytePointer result = engine.GetUTF8Text(); + if (result != null) { + LOGGER.info("Tesseract has found: {}", result.getString()); + result.deallocate(); + } else { + LOGGER.warn("Failed to analyze image"); + } + + } + + @Override + public void Destroy() { + engine.End(); + LOGGER.info("Tesseract engine destroyed"); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java new file mode 100644 index 000000000..a02a119cb --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java @@ -0,0 +1,30 @@ +package nl.ou.testar.visualvalidation.ocr.tesseract; + +import org.testar.settings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class TesseractSettings extends ExtendedSettingBase { + public String dataPath; + public String language; + + public static TesseractSettings CreateDefault() { + TesseractSettings instance = new TesseractSettings(); + instance.dataPath = System.getenv("LOCALAPPDATA") + "\\Tesseract-OCR\\tessdata"; + instance.language = "eng"; + return instance; + } + + @Override + public int compareTo(TesseractSettings other) { + int result = -1; + if ((language.contentEquals(other.language) && (dataPath.contentEquals(other.dataPath)))) { + result = 0; + } + return result; + } +} diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index a352bcb5c..4d5b03534 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -42,6 +42,7 @@ import static org.fruit.monkey.ConfigTags.LogLevel; import java.awt.Desktop; +import java.awt.image.BufferedImage; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; @@ -64,6 +65,8 @@ import es.upv.staq.testar.*; import nl.ou.testar.*; import nl.ou.testar.HtmlReporting.Reporting; +import nl.ou.testar.visualvalidation.VisualValidationFactory; +import nl.ou.testar.visualvalidation.VisualValidationManager; import nl.ou.testar.StateModel.StateModelManager; import nl.ou.testar.StateModel.StateModelManagerFactory; import org.apache.logging.log4j.LogManager; @@ -190,6 +193,7 @@ protected final double timeElapsed() { BluePen = Pen.newPen().setColor(Color.Blue). setFillPattern(FillPattern.None).setStrokePattern(StrokePattern.Solid).build(); + protected VisualValidationManager visualValidationManager; /** * This is the abstract flow of TESTAR (generate mode): @@ -357,6 +361,8 @@ protected void initialize(Settings settings) { // new state model manager stateModelManager = StateModelManagerFactory.getStateModelManager(settings); + + visualValidationManager = VisualValidationFactory.createVisualValidator(); } try { @@ -1499,6 +1505,11 @@ private void setStateScreenshot(State state) { //System.out.println("DEBUG: normal state shot"); state.set(Tags.ScreenshotPath, ProtocolUtil.getStateshot(state)); } + AWTCanvas screenShot = protocolUtil.getStateshotBinary(state); + visualValidationManager.AnalyzeImage(screenShot.image()); + String screenshotPath = ScreenshotSerialiser.saveStateshot(state.get(Tags.ConcreteIDCustom, + "NoConcreteIdAvailable"), screenShot); + state.set(Tags.ScreenshotPath, screenshotPath); } } @@ -1837,6 +1848,7 @@ private void closeTestarTestSession(){ GlobalScreen.unregisterNativeHook(); } } + visualValidationManager.Close(); } catch(Exception e) { e.printStackTrace(); } diff --git a/testar/src/org/testar/settings/ExtendedSettingBase.java b/testar/src/org/testar/settings/ExtendedSettingBase.java index 4673220ee..ff5d0cdc5 100644 --- a/testar/src/org/testar/settings/ExtendedSettingBase.java +++ b/testar/src/org/testar/settings/ExtendedSettingBase.java @@ -36,7 +36,7 @@ public abstract class ExtendedSettingBase extends Observable implements IExtendedSetting, Comparable, Serializable { /** - * Notify the {@link IExtendedSettingContainer} that the specialization of this class needs to be saved. + * Notify the {@link ExtendedSettingContainer} that the specialization of this class needs to be saved. */ @Override public void Save() { diff --git a/testar/src/org/testar/settings/ExtendedSettingsFactory.java b/testar/src/org/testar/settings/ExtendedSettingsFactory.java index c868a14a4..c72be9687 100644 --- a/testar/src/org/testar/settings/ExtendedSettingsFactory.java +++ b/testar/src/org/testar/settings/ExtendedSettingsFactory.java @@ -31,6 +31,7 @@ package org.testar.settings; import nl.ou.testar.visualvalidation.VisualValidationSettings; +import nl.ou.testar.visualvalidation.ocr.tesseract.TesseractSettings; import java.util.ArrayList; import java.util.List; @@ -84,4 +85,8 @@ public static VisualValidationSettings createVisualValidationSettings() { public static ExampleSetting createTestSetting() { return createSettings(ExampleSetting.class, ExampleSetting::CreateDefault); } -} \ No newline at end of file + + public static TesseractSettings createTesseractSetting() { + return createSettings(TesseractSettings.class, TesseractSettings::CreateDefault); + } +} From 36ecc750e625ef1d191b2dd2b4f602d8481e6228 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Tue, 8 Dec 2020 21:50:19 +0100 Subject: [PATCH 03/40] Added result extraction. --- .../ocr/RecognizedElement.java | 44 ++++++++++++++++ .../ocr/tesseract/TesseractOcrEngine.java | 38 ++++++++++---- .../ocr/tesseract/TesseractResult.java | 50 +++++++++++++++++++ 3 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java new file mode 100644 index 000000000..d9a063615 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java @@ -0,0 +1,44 @@ +package nl.ou.testar.visualvalidation.ocr; + +/** + * A discovered text element by an OCR engine. + */ +public class RecognizedElement { + final int _x1; + final int _y1; + final int _x2; + final int _y2; + final float _confidence; + final String _text; + + /** + * Constructor. + * + * @param x1 The first X coordinate of the discovered text. + * @param y1 The first Y coordinate of the discovered text. + * @param x2 The second X coordinate of the discovered text. + * @param y2 The second Y coordinate of the discovered text. + * @param confidence The confidence level of the discovered text. + * @param text The discovered text. + */ + public RecognizedElement(int x1, int y1, int x2, int y2, float confidence, String text) { + _x1 = x1; + _y1 = y1; + _x2 = x2; + _y2 = y2; + _confidence = confidence; + _text = text; + } + + @Override + public String toString() { + return "RecognizedElement{" + + "_x1=" + _x1 + + ", _y1=" + _y1 + + ", _x2=" + _x2 + + ", _y2=" + _y2 + + ", _confidence=" + _confidence + + ", _text='" + _text + '\'' + + '}'; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java index 26a4709e3..8c92ffb56 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java @@ -1,10 +1,13 @@ package nl.ou.testar.visualvalidation.ocr.tesseract; import nl.ou.testar.visualvalidation.ocr.OcrEngineInterface; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.tesseract.ETEXT_DESC; +import org.bytedeco.tesseract.ResultIterator; import org.bytedeco.tesseract.TessBaseAPI; +import org.bytedeco.tesseract.global.tesseract; import org.testar.settings.ExtendedSettingsFactory; import java.awt.image.BufferedImage; @@ -16,6 +19,8 @@ import java.nio.ByteBuffer; import java.nio.IntBuffer; import java.nio.ShortBuffer; +import java.util.ArrayList; +import java.util.List; public class TesseractOcrEngine implements OcrEngineInterface { private final TessBaseAPI engine; @@ -35,6 +40,27 @@ public TesseractOcrEngine() { @Override public void ScanImage(BufferedImage image) { // TODO TM: Image analysis should be done on separate thread. + loadImage(image); + + if (engine.Recognize(new ETEXT_DESC()) != 0) { + // TODO TM: Should we throw or just log and proceed with the application and set the matcher result to unknown-ish + throw new IllegalArgumentException("could not recognize text"); + } + + List recognizedWords = new ArrayList<>(); + try (ResultIterator recognizedElement = engine.GetIterator()) { + int level = tesseract.RIL_WORD; + do { + recognizedWords.add(TesseractResult.Extract(recognizedElement, level)); + } while (recognizedElement.Next(level)); + + recognizedWords.forEach(ocrWord -> LOGGER.info("Found {}", ocrWord)); + } + + engine.Clear(); + } + + private void loadImage(BufferedImage image) { DataBuffer dataBuffer = image.getData().getDataBuffer(); ByteBuffer byteBuffer; @@ -62,14 +88,8 @@ public void ScanImage(BufferedImage image) { int bytes_per_line = bytes_per_pixel * image.getWidth(); engine.SetImage(byteBuffer, image.getWidth(), image.getHeight(), bytes_per_pixel, bytes_per_line); - BytePointer result = engine.GetUTF8Text(); - if (result != null) { - LOGGER.info("Tesseract has found: {}", result.getString()); - result.deallocate(); - } else { - LOGGER.warn("Failed to analyze image"); - } - + // TODO TM: Figure out which value we should use. + engine.SetSourceResolution(160); } @Override diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java new file mode 100644 index 000000000..843e77d34 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java @@ -0,0 +1,50 @@ +package nl.ou.testar.visualvalidation.ocr.tesseract; + +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.tesseract.ResultIterator; + +import java.util.function.Supplier; + +/** + * Convert a tesseract result into a {@link RecognizedElement} + */ +public class TesseractResult { + + /** + * Convert a {@link TesseractResult} into a {@link RecognizedElement} + * + * @param recognizedElement The recognized element. + * @param granularity The granularity of recognized element, range from a block till individual character. + * See tesseract::PageIteratorLevel for more information. + * @return An {@link RecognizedElement}. + */ + static RecognizedElement Extract(ResultIterator recognizedElement, int granularity) { + Supplier intPointerSupplier = () -> new IntPointer(new int[1]); + + BytePointer ocrResult = recognizedElement.GetUTF8Text(granularity); + String recognizedText = ocrResult.getString().trim(); + + float confidence = recognizedElement.Confidence(granularity); + IntPointer x1 = intPointerSupplier.get(); + IntPointer y1 = intPointerSupplier.get(); + IntPointer x2 = intPointerSupplier.get(); + IntPointer y2 = intPointerSupplier.get(); + boolean foundRectangle = recognizedElement.BoundingBox(granularity, x1, y1, x2, y2); + + if (!foundRectangle) { + throw new IllegalArgumentException("Could not find any rectangle for this element"); + } + + RecognizedElement result = new RecognizedElement(x1.get(), y1.get(), x2.get(), y2.get(), confidence, recognizedText); + + x1.deallocate(); + y1.deallocate(); + x2.deallocate(); + y2.deallocate(); + ocrResult.deallocate(); + + return result; + } +} From dd5a30f2b2a0c4e5d924ad1cf3ade1a48f317d2d Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 11 Dec 2020 11:24:40 +0100 Subject: [PATCH 04/40] Added log4j facade --- .../src/org/fruit/monkey/DefaultProtocol.java | 1 - testar/src/org/testar/Logger.java | 17 +++++++++++++ .../org/testar/protocols/DesktopProtocol.java | 24 +++++++++++-------- .../testar/protocols/WebdriverProtocol.java | 19 ++++++++------- .../testar/settings/ExtendedSettingFile.java | 16 ++++++------- 5 files changed, 50 insertions(+), 27 deletions(-) create mode 100644 testar/src/org/testar/Logger.java diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index 4d5b03534..b8ad2d9e9 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -162,7 +162,6 @@ protected final double timeElapsed() { protected List contextRunningProcesses = null; protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; - protected static final Logger INDEXLOG = LogManager.getLogger(); protected double passSeverity = Verdict.SEVERITY_OK; public static Action lastExecutedAction = null; diff --git a/testar/src/org/testar/Logger.java b/testar/src/org/testar/Logger.java new file mode 100644 index 000000000..14adeb77d --- /dev/null +++ b/testar/src/org/testar/Logger.java @@ -0,0 +1,17 @@ +package org.testar; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; + +/** + * Facade for the log4j logger. + * Reduces the need to initialize the logger in every class. + * By wrapping {@code log4j.logger.log} with a tag argument a more uniformed log is realized. + */ +public class Logger { + private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(); + + public static void log(Level level, String tag, String message, Object... params) { + LOGGER.log(level, "[" + tag + "] " + message, params); + } +} diff --git a/testar/src/org/testar/protocols/DesktopProtocol.java b/testar/src/org/testar/protocols/DesktopProtocol.java index 145744f07..ea99bc868 100644 --- a/testar/src/org/testar/protocols/DesktopProtocol.java +++ b/testar/src/org/testar/protocols/DesktopProtocol.java @@ -34,6 +34,7 @@ import nl.ou.testar.DerivedActions; import nl.ou.testar.HtmlReporting.Reporting; import nl.ou.testar.RandomActionSelector; +import org.apache.logging.log4j.Level; import org.fruit.Drag; import org.fruit.Environment; import org.fruit.alayer.*; @@ -42,14 +43,17 @@ import org.fruit.alayer.exceptions.ActionBuildException; import org.fruit.alayer.exceptions.StateBuildException; import org.fruit.monkey.ConfigTags; +import org.testar.Logger; import org.testar.OutputStructure; import java.io.File; import java.util.HashSet; import java.util.Set; + import static org.fruit.alayer.Tags.Blocked; import static org.fruit.alayer.Tags.Enabled; public class DesktopProtocol extends GenericUtilsProtocol { + private static final String TAG = "DesktopProtocol"; //Attributes for adding slide actions protected static double SCROLL_ARROW_SIZE = 36; // sliding arrows protected static double SCROLL_THICK = 16; //scroll thickness @@ -198,16 +202,16 @@ protected void postSequenceProcessing() { try { sequencesPath = new File(getGeneratedSequenceName()).getCanonicalPath(); }catch (Exception e) {} - - statusInfo = statusInfo.replace("\n"+Verdict.OK.info(), ""); - - //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" - INDEXLOG.info(OutputStructure.executedSUTname - + " " + settings.get(ConfigTags.Mode, mode()) - + " " + sequencesPath - + " " + status + " \"" + statusInfo + "\"" ); - - htmlReport.close(); + + statusInfo = statusInfo.replace("\n"+Verdict.OK.info(), ""); + + //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" + Logger.log(Level.INFO, TAG, OutputStructure.executedSUTname + + " " + settings.get(ConfigTags.Mode, mode()) + + " " + sequencesPath + + " " + status + " \"" + statusInfo + "\"" ); + + htmlReport.close(); } /** diff --git a/testar/src/org/testar/protocols/WebdriverProtocol.java b/testar/src/org/testar/protocols/WebdriverProtocol.java index bc084f3c4..4ba906046 100644 --- a/testar/src/org/testar/protocols/WebdriverProtocol.java +++ b/testar/src/org/testar/protocols/WebdriverProtocol.java @@ -49,6 +49,7 @@ import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.Level; import org.fruit.Environment; import org.fruit.Pair; import org.fruit.alayer.Action; @@ -69,6 +70,7 @@ import org.fruit.alayer.windows.Windows; import org.fruit.monkey.ConfigTags; import org.fruit.monkey.Settings; +import org.testar.Logger; import org.testar.OutputStructure; import es.upv.staq.testar.NativeLinker; @@ -76,7 +78,8 @@ import nl.ou.testar.HtmlReporting.Reporting; public class WebdriverProtocol extends GenericUtilsProtocol { - //Attributes for adding slide actions + private static final String TAG = "WebdriverProtocol"; + //Attributes for adding slide actions protected static double SCROLL_ARROW_SIZE = 36; // sliding arrows protected static double SCROLL_THICK = 16; //scroll thickness protected Reporting htmlReport; @@ -371,13 +374,13 @@ protected void postSequenceProcessing() { statusInfo = statusInfo.replace("\n"+Verdict.OK.info(), ""); - //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" - INDEXLOG.info(OutputStructure.executedSUTname - + " " + settings.get(ConfigTags.Mode, mode()) - + " " + sequencesPath - + " " + status + " \"" + statusInfo + "\"" ); - - htmlReport.close(); + //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" + Logger.log(Level.INFO, TAG, OutputStructure.executedSUTname + + " " + settings.get(ConfigTags.Mode, mode()) + + " " + sequencesPath + + " " + status + " \"" + statusInfo + "\"" ); + + htmlReport.close(); } @Override diff --git a/testar/src/org/testar/settings/ExtendedSettingFile.java b/testar/src/org/testar/settings/ExtendedSettingFile.java index 6aab5edca..b3b3dffda 100644 --- a/testar/src/org/testar/settings/ExtendedSettingFile.java +++ b/testar/src/org/testar/settings/ExtendedSettingFile.java @@ -32,9 +32,9 @@ import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.SerializationUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Level; import org.checkerframework.checker.nullness.qual.NonNull; +import org.testar.Logger; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; @@ -90,7 +90,7 @@ public ExtractionResult(JAXBContext context, RootSetting data, Boolean fileNotFo } public class ExtendedSettingFile implements Serializable { - private static final Logger LOGGER = LogManager.getLogger(); + private static final String TAG = "ExtendedSettings"; public static final String FileName = "ExtendedSettings.xml"; private final String _absolutePath; private final ReentrantReadWriteLock _fileAccessMutex; @@ -137,14 +137,14 @@ public T load(@NonNull Class clazz) { // We only support loading a single element for now. if (rd.Data.any.stream().filter(element -> element.getClass() == clazz).count() > 1) { - LOGGER.error("Duplicate elements found for {}, returning first element ", clazz); + Logger.log(Level.ERROR, TAG, "Duplicate elements found for {}, returning first element ", clazz); } // Store the current content, so we can replace it when needed. _loadedValue = SerializationUtils.clone((Serializable) result); } if (result == null) { - LOGGER.info("Did not found XML element for class: {}", clazz); + Logger.log(Level.INFO, TAG,"Did not found XML element for class: {}", clazz); } return result; @@ -163,7 +163,7 @@ public T load(@NonNull Class clazz, @NonNull IExtendedSettingDefaultValue T result = load(clazz); if (result == null) { - LOGGER.info("Writing default values for {}", clazz); + Logger.log(Level.TRACE, TAG,"Writing default values for {}", clazz); save(defaultFunctor.CreateDefault()); return load(clazz); } @@ -179,7 +179,7 @@ public T load(@NonNull Class clazz, @NonNull IExtendedSettingDefaultValue */ public void save(@NonNull Object data) { if (!(data instanceof Comparable)) { - LOGGER.error("Object {} is not extending Comparable", data); + Logger.log(Level.ERROR, TAG,"Object {} is not extending Comparable", data); return; } @@ -281,7 +281,7 @@ private void createExtendedSettingsFile() { Marshaller marshaller = context.createMarshaller(); marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); marshaller.marshal(new RootSetting(), os); - LOGGER.info("Created extended settings file: {}", _absolutePath); + Logger.log(Level.INFO, TAG,"Created extended settings file: {}", _absolutePath); } catch (JAXBException e) { e.printStackTrace(); } finally { From 6ea63ad52637bea4ddc9bd1d9f8eae9a6553006e Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 11 Dec 2020 13:21:06 +0100 Subject: [PATCH 05/40] Made download link to chromedriver clickable. --- .../src/org/fruit/monkey/DefaultProtocol.java | 113 ++++++++++++------ 1 file changed, 77 insertions(+), 36 deletions(-) diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index b8ad2d9e9..e4473ec23 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -41,8 +41,7 @@ import static org.fruit.alayer.Tags.SystemState; import static org.fruit.monkey.ConfigTags.LogLevel; -import java.awt.Desktop; -import java.awt.image.BufferedImage; +import java.awt.*; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; @@ -53,14 +52,18 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintStream; +import java.net.URI; +import java.net.URISyntaxException; import java.util.*; +import java.util.List; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import javax.swing.JFrame; -import javax.swing.JOptionPane; +import javax.swing.*; +import javax.swing.event.HyperlinkEvent; +import javax.swing.event.HyperlinkListener; import es.upv.staq.testar.*; import nl.ou.testar.*; @@ -69,12 +72,14 @@ import nl.ou.testar.visualvalidation.VisualValidationManager; import nl.ou.testar.StateModel.StateModelManager; import nl.ou.testar.StateModel.StateModelManagerFactory; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.fruit.Assert; import org.fruit.Pair; import org.fruit.Util; import org.fruit.alayer.*; +import org.fruit.alayer.Action; +import org.fruit.alayer.Canvas; +import org.fruit.alayer.Color; +import org.fruit.alayer.Shape; import org.fruit.alayer.actions.*; import org.fruit.alayer.devices.AWTMouse; import org.fruit.alayer.devices.KBKeys; @@ -178,7 +183,7 @@ protected final double timeElapsed() { protected boolean forceToForeground = false; protected int testFailTimes = 0; protected boolean nonSuitableAction = false; - + protected int escAttempts = 0; protected static final int MAX_ESC_ATTEMPTS = 99; @@ -278,31 +283,29 @@ public final void run(final Settings settings) { System.out.println(msg); this.mode = Modes.Quit; - + }catch(SessionNotCreatedException e) { if(e.getMessage().contains("Chrome version")) { - - String msg = "*** Unsupported versions exception: Chrome browser and Selenium WebDriver versions *** \n" - + "Please verify your Chrome browser version: chrome://settings/help \n" - + "And download the appropiate ChromeDriver version: https://chromedriver.chromium.org/downloads \n" - + "\n" - + "Surely exists a residual process \"chromedriver.exe\" running. \n" - + "You can use Task Manager to finish it."; - - popupMessage(msg); - + String msg = "*** Unsupported versions exception: Chrome browser and Selenium WebDriver versions ***" + + "
Please verify your Chrome browser version: chrome://settings/help" + + "
And download the appropriate ChromeDriver version." + + "
" + + "
Surely exists a residual process \"chromedriver.exe\" running." + + "
You can use Task Manager to finish it."; + + chromeDriverMissing(msg); + System.out.println(msg); - System.out.println(e.getMessage()); - - }else { + + }else { System.out.println("********** ERROR starting Selenium WebDriver ********"); - System.out.println(e.getMessage()); - } - + } + System.out.println(e.getMessage()); + }catch (IllegalStateException e) { if (e.getMessage().contains("driver executable does not exist")) { - + String msg = "Exception: Check if chromedriver.exe path: \n" +settings.get(ConfigTags.SUTConnectorValue) +"\n exists or if is a correct definition"; @@ -310,11 +313,11 @@ public final void run(final Settings settings) { popupMessage(msg); System.out.println(msg); - + }else { e.printStackTrace(); } - + }catch(SystemStartException SystemStartException) { SystemStartException.printStackTrace(); this.mode = Modes.Quit; @@ -468,6 +471,47 @@ private void popupMessage(String message) { if(settings.get(ConfigTags.ShowVisualSettingsDialogOnStartup)) { JFrame frame = new JFrame(); JOptionPane.showMessageDialog(frame, message); + + } + } + + /** + * Show a popup containing a html message with click interaction. + * Only if GUI option is enabled (disabled for CI) + */ + private void chromeDriverMissing(String htmlMessage) { + if(settings.get(ConfigTags.ShowVisualSettingsDialogOnStartup)) { + JFrame frame = new JFrame(); + // for copying style + JLabel label = new JLabel(); + Font font = label.getFont(); + + // create some css from the label's font + String style = "font-family:" + font.getFamily() + ";" + + "font-weight:" + (font.isBold() ? "bold" : "normal") + ";" + + "font-size:" + font.getSize() + "pt;"; + // html content + JEditorPane ep = new JEditorPane("text/html", "" + + htmlMessage + + ""); + + // handle link events + ep.addHyperlinkListener(e -> { + if (e.getEventType().equals(HyperlinkEvent.EventType.ACTIVATED)) { + Desktop desktop = Desktop.getDesktop(); + try { + desktop.browse(e.getURL().toURI()); + } catch (URISyntaxException | IOException exception) { + exception.printStackTrace(); + } + } + }); + ep.setEditable(false); + ep.setBackground(label.getBackground()); + + // show + JOptionPane.showMessageDialog(frame, ep); + } } @@ -700,7 +744,7 @@ protected void runGenerateOuterLoop(SUT system) { // beginSequence() - a script to interact with GUI, for example login screen LogSerialiser.log("Starting sequence " + sequenceCount + " (output as: " + generatedSequence + ")\n\n", LogSerialiser.LogLevel.Info); beginSequence(system, state); - + //update state after begin sequence SUT modification state = getState(system); @@ -943,17 +987,16 @@ protected void runSpyLoop() { Set actions = deriveActions(system,state); buildStateActionsIdentifiers(state, actions); - //in Spy-mode, always visualize the widget info under the mouse cursor: SutVisualization.visualizeState(visualizationOn, markParentWidget, mouse, lastPrintParentsOf, cv, state); //in Spy-mode, always visualize the green dots: visualizeActions(cv, state, actions); - + cv.end(); int msRefresh = (int)(settings.get(ConfigTags.RefreshSpyCanvas, 0.5) * 1000); - + synchronized (this) { try { this.wait(msRefresh); @@ -971,7 +1014,7 @@ protected void runSpyLoop() { Util.clear(cv); cv.end(); - + //finishSequence() content, but SPY mode is not a sequence if(!NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { SystemProcessHandling.killTestLaunchedProcesses(this.contextRunningProcesses); @@ -1487,7 +1530,7 @@ protected State getState(SUT system) throws StateBuildException { passSeverity = verdict.severity(); LogSerialiser.log("Detected warning: " + verdict + "\n", LogSerialiser.LogLevel.Critical); } - + return state; } @@ -1551,7 +1594,7 @@ protected Verdict getVerdict(State state){ // if everything was OK ... return Verdict.OK; } - + private Verdict suspiciousStringValueMatcher(Widget w) { Matcher m; @@ -1692,10 +1735,8 @@ else if (actions.isEmpty()){ protected boolean executeAction(SUT system, State state, Action action){ if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)){ - //System.out.println("DEBUG: Using WebDriver specific action shot."); WdProtocolUtil.getActionshot(state,action); }else{ - //System.out.println("DEBUG: normal action shot"); ProtocolUtil.getActionshot(state,action); } From ac7bd377ef93b818f6b7c196e8d43998771cba36 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 11 Dec 2020 15:59:54 +0100 Subject: [PATCH 06/40] Added initial OCR functionality. --- .../DummyVisualValidator.java | 8 +- .../VisualValidationManager.java | 16 ++- .../visualvalidation/VisualValidator.java | 82 ++++++++++- .../ocr/OcrEngineInterface.java | 11 +- .../ocr/OcrResultCallback.java | 7 + .../ocr/dummy/DummyOcrEngine.java | 7 +- .../ocr/tesseract/TesseractOcrEngine.java | 127 ++++++++++++++---- .../ocr/tesseract/TesseractSettings.java | 2 + .../src/org/fruit/monkey/DefaultProtocol.java | 16 ++- 9 files changed, 228 insertions(+), 48 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java diff --git a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java index e695940db..547ea3959 100644 --- a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java @@ -1,15 +1,19 @@ package nl.ou.testar.visualvalidation; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.fruit.alayer.AWTCanvas; +import org.fruit.alayer.State; + import java.awt.image.BufferedImage; public class DummyVisualValidator implements VisualValidationManager{ @Override - public void AnalyzeImage(BufferedImage image) { + public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { } @Override - public void Close() { + public void Destroy() { } } diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java index 01273cc6f..e5eb6a821 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java @@ -1,9 +1,19 @@ package nl.ou.testar.visualvalidation; -import java.awt.image.BufferedImage; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.fruit.alayer.AWTCanvas; +import org.fruit.alayer.State; public interface VisualValidationManager { - void AnalyzeImage(BufferedImage image); + /** + * Analyze the captured image and update the verdict. + * @param state The state of the application. + * @param screenshot The captured screenshot of the current state. + */ + void AnalyzeImage(State state, @Nullable AWTCanvas screenshot); - void Close(); + /** + * Destroy the visual validation manager. + */ + void Destroy(); } diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 0804c102e..89eaa1ef2 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -5,12 +5,24 @@ import nl.ou.testar.visualvalidation.ocr.OcrConfiguration; import nl.ou.testar.visualvalidation.ocr.OcrEngineFactory; import nl.ou.testar.visualvalidation.ocr.OcrEngineInterface; +import nl.ou.testar.visualvalidation.ocr.OcrResultCallback; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; +import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.fruit.alayer.AWTCanvas; +import org.fruit.alayer.State; +import org.testar.Logger; -import java.awt.image.BufferedImage; +import java.util.List; -public class VisualValidator implements VisualValidationManager { +public class VisualValidator implements VisualValidationManager, OcrResultCallback { private final OcrEngineInterface ocrEngine; private final VisualMatcher matcher; + private final String TAG = "VisualValidator"; + private int analysisId = 0; + private final Boolean _ocrResultReceived = false; + private final Boolean _expectedTextReceived = false; + private List _ocrItems = null; public VisualValidator(VisualValidationSettings settings) { OcrConfiguration ocrConfig = settings.ocrConfiguration; @@ -24,16 +36,76 @@ public VisualValidator(VisualValidationSettings settings) { } @Override - public void AnalyzeImage(BufferedImage image) { + public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { + // Create new session + startNewAnalysis(); - ocrEngine.ScanImage(image); + // Start ocr analysis, provide callback once finished. + parseScreenshot(screenshot); + + // Start extracting text, provide callback once finished. + extractExpectedText(state); + + // Wait for both results. + matchText(); + + updateVerdict(state); + + storeAnalysis(); + } + + private void startNewAnalysis() { + analysisId++; + Logger.log(Level.INFO, TAG, "Starting new analysis {}", analysisId); + } + + private void parseScreenshot(@Nullable AWTCanvas screenshot) { + if (screenshot != null) { + ocrEngine.AnalyzeImage(screenshot.image(), this); + + } else { + Logger.log(Level.ERROR, TAG, "No screenshot for current state"); + } + } + + private void extractExpectedText(State state) { + Logger.log(Level.INFO, TAG, "Extracting text"); + } + + private void matchText() { + synchronized (_ocrResultReceived) { + try { + _ocrResultReceived.wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + Logger.log(Level.INFO, TAG, "Matching {} with {}", _ocrItems, "api"); + } + + private void updateVerdict(State state) { + Logger.log(Level.INFO, TAG, "Updating verdict {}"); + } + + private void storeAnalysis() { + Logger.log(Level.INFO, TAG, "Storing analysis"); } @Override - public void Close() { + public void Destroy() { matcher.destroy(); if (ocrEngine != null) { ocrEngine.Destroy(); } } + + @Override + public void reportResult(List items) { + synchronized (_ocrResultReceived) { + _ocrItems = items; + _ocrResultReceived.notify(); + Logger.log(Level.INFO, TAG, "Received {} result", items.size()); + } + } } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java index 3ce87a655..9fa2a16f3 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrEngineInterface.java @@ -3,7 +3,16 @@ import java.awt.image.BufferedImage; public interface OcrEngineInterface { - void ScanImage(BufferedImage image); + /** + * Analyze the given image with an OCR engine and return the detected text via the callback. + * + * @param image The image we want to analyze. + * @param callback Callback function for returning the detected text. + */ + void AnalyzeImage(BufferedImage image, OcrResultCallback callback); + /** + * Destroy the OCR engine. + */ void Destroy(); } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java new file mode 100644 index 000000000..b15c26d2b --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java @@ -0,0 +1,7 @@ +package nl.ou.testar.visualvalidation.ocr; + +import java.util.List; + +public interface OcrResultCallback { + void reportResult(List items); +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java index a335938e5..a35f341a4 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java @@ -1,13 +1,16 @@ package nl.ou.testar.visualvalidation.ocr.dummy; import nl.ou.testar.visualvalidation.ocr.OcrEngineInterface; +import nl.ou.testar.visualvalidation.ocr.OcrResultCallback; import java.awt.image.BufferedImage; +import java.util.ArrayList; public class DummyOcrEngine implements OcrEngineInterface { @Override - public void ScanImage(BufferedImage image) { - + public void AnalyzeImage(BufferedImage image, OcrResultCallback callback) { + // Immediately trigger the callback. + callback.reportResult(new ArrayList<>()); } @Override diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java index 8c92ffb56..d45b37364 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java @@ -1,13 +1,14 @@ package nl.ou.testar.visualvalidation.ocr.tesseract; import nl.ou.testar.visualvalidation.ocr.OcrEngineInterface; +import nl.ou.testar.visualvalidation.ocr.OcrResultCallback; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Level; import org.bytedeco.tesseract.ETEXT_DESC; import org.bytedeco.tesseract.ResultIterator; import org.bytedeco.tesseract.TessBaseAPI; import org.bytedeco.tesseract.global.tesseract; +import org.testar.Logger; import org.testar.settings.ExtendedSettingsFactory; import java.awt.image.BufferedImage; @@ -21,46 +22,100 @@ import java.nio.ShortBuffer; import java.util.ArrayList; import java.util.List; - -public class TesseractOcrEngine implements OcrEngineInterface { - private final TessBaseAPI engine; - static final Logger LOGGER = LogManager.getLogger(); +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * OCR engine implementation dedicated for the Tesseract engine. + * Analyzes the given image via {@link #AnalyzeImage} in a separate thread. + * Once the engine has finished the caller shall be informed via the given callback. + * + * The current implementation can only process one image at the time. This is enforced by the synchronized statements in + * {@link #AnalyzeImage} where we set the buffer which is used by the engine and the actual execution of the engine in + * the {@link #run}. + */ +public class TesseractOcrEngine extends Thread implements OcrEngineInterface { + private static final String TAG = "Tesseract"; + + AtomicBoolean running = new AtomicBoolean(true); + + private final TessBaseAPI _engine; + private final Boolean _scanSync = true; + private final int _imageResolution; + private BufferedImage _image = null; + private OcrResultCallback _callback = null; public TesseractOcrEngine() { - engine = new TessBaseAPI(); + _engine = new TessBaseAPI(); TesseractSettings config = ExtendedSettingsFactory.createTesseractSetting(); + _imageResolution = config.imageResolution; - if (engine.Init(config.dataPath, config.language) != 0) { - LOGGER.error("Could not initialize tesseract."); + if (_engine.Init(config.dataPath, config.language) != 0) { + Logger.log(Level.ERROR, TAG, "Could not initialize tesseract."); } - LOGGER.info("Tesseract engine created; Language:{} Data path:{}", config.language, config.dataPath); + Logger.log(Level.INFO, TAG, "Tesseract engine created; Language:{} Data path:{}", + config.language, config.dataPath); + + setName(TAG); + start(); } @Override - public void ScanImage(BufferedImage image) { - // TODO TM: Image analysis should be done on separate thread. - loadImage(image); + public void AnalyzeImage(BufferedImage image, OcrResultCallback callback) { + synchronized (_scanSync) { + _image = image; + _callback = callback; + Logger.log(Level.TRACE, TAG, "Queue new image scan."); + _scanSync.notify(); + } + } - if (engine.Recognize(new ETEXT_DESC()) != 0) { - // TODO TM: Should we throw or just log and proceed with the application and set the matcher result to unknown-ish - throw new IllegalArgumentException("could not recognize text"); + @Override + public void run() { + while (running.get()) { + synchronized (_scanSync) { + try { + // Wait until we need to inspect a new image. + _scanSync.wait(); + if (!running.get()) { + break; + } + recognizeText(); + + } catch (InterruptedException e) { + // Happens if someone interrupts your thread. + Logger.log(Level.INFO, TAG, "Wait interrupted"); + e.printStackTrace(); + } + } } + } + private void recognizeText() { List recognizedWords = new ArrayList<>(); - try (ResultIterator recognizedElement = engine.GetIterator()) { - int level = tesseract.RIL_WORD; - do { - recognizedWords.add(TesseractResult.Extract(recognizedElement, level)); - } while (recognizedElement.Next(level)); - recognizedWords.forEach(ocrWord -> LOGGER.info("Found {}", ocrWord)); + loadImageIntoEngine(_image); + + if (_engine.Recognize(new ETEXT_DESC()) != 0) { + Logger.log(Level.ERROR, TAG, "Could not process image."); + } else { + try (ResultIterator recognizedElement = _engine.GetIterator()) { + int level = tesseract.RIL_WORD; + do { + recognizedWords.add(TesseractResult.Extract(recognizedElement, level)); + } while (recognizedElement.Next(level)); + + recognizedWords.forEach(ocrWord -> Logger.log(Level.DEBUG, TAG, "Found {}", ocrWord)); + } } + _engine.Clear(); + // Notify the callback with the discovered words. + _callback.reportResult(recognizedWords); - engine.Clear(); + Logger.log(Level.DEBUG, TAG, "Finished image scan found {} elements.", recognizedWords.size()); } - private void loadImage(BufferedImage image) { + private void loadImageIntoEngine(BufferedImage image) { DataBuffer dataBuffer = image.getData().getDataBuffer(); ByteBuffer byteBuffer; @@ -87,14 +142,28 @@ private void loadImage(BufferedImage image) { int bytes_per_pixel = image.getColorModel().getNumComponents(); int bytes_per_line = bytes_per_pixel * image.getWidth(); - engine.SetImage(byteBuffer, image.getWidth(), image.getHeight(), bytes_per_pixel, bytes_per_line); - // TODO TM: Figure out which value we should use. - engine.SetSourceResolution(160); + _engine.SetImage(byteBuffer, image.getWidth(), image.getHeight(), bytes_per_pixel, bytes_per_line); + _engine.SetSourceResolution(_imageResolution); } @Override public void Destroy() { - engine.End(); - LOGGER.info("Tesseract engine destroyed"); + stopAndJoinThread(); + + _engine.End(); + Logger.log(Level.DEBUG, TAG, "Engine destroyed."); + } + + private void stopAndJoinThread() { + synchronized (_scanSync) { + running.set(false); + _scanSync.notify(); + } + + try { + join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java index a02a119cb..ead0f49c6 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java @@ -11,11 +11,13 @@ public class TesseractSettings extends ExtendedSettingBase { public String dataPath; public String language; + public int imageResolution; public static TesseractSettings CreateDefault() { TesseractSettings instance = new TesseractSettings(); instance.dataPath = System.getenv("LOCALAPPDATA") + "\\Tesseract-OCR\\tessdata"; instance.language = "eng"; + instance.imageResolution = 160; return instance; } diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index e4473ec23..d59d91bc7 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -1509,14 +1509,15 @@ protected State getState(SUT system) throws StateBuildException { setStateForClickFilterLayerProtocol(state); - if(settings.get(ConfigTags.Mode) == Modes.Spy) + if(settings.get(ConfigTags.Mode) == Modes.Spy) { return state; - - Verdict verdict = getVerdict(state); - state.set(Tags.OracleVerdict, verdict); + } setStateScreenshot(state); + Verdict verdict = getVerdict(state); + state.set(Tags.OracleVerdict, verdict); + if (mode() != Modes.Spy && verdict.severity() >= settings().get(ConfigTags.FaultThreshold)){ faultySequence = true; LogSerialiser.log("Detected fault: " + verdict + "\n", LogSerialiser.LogLevel.Critical); @@ -1535,10 +1536,12 @@ protected State getState(SUT system) throws StateBuildException { } /** - * Take a Screenshot of the State and associate the path into state tag + * Take a Screenshot of the State and associate the path into state tag. + * If enabled run the visual validation on the capture screenshot. */ private void setStateScreenshot(State state) { Shape viewPort = state.get(Tags.Shape, null); + AWTCanvas screenShot = null; if(viewPort != null){ if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)){ //System.out.println("DEBUG: Using WebDriver specific state shot."); @@ -1553,6 +1556,7 @@ private void setStateScreenshot(State state) { "NoConcreteIdAvailable"), screenShot); state.set(Tags.ScreenshotPath, screenshotPath); } + visualValidationManager.AnalyzeImage(state, screenShot); } @Override @@ -1888,7 +1892,7 @@ private void closeTestarTestSession(){ GlobalScreen.unregisterNativeHook(); } } - visualValidationManager.Close(); + visualValidationManager.Destroy(); } catch(Exception e) { e.printStackTrace(); } From aeeb193ec04b1add2333c32ac7e27c41f7fde339 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 12 Dec 2020 11:32:03 +0100 Subject: [PATCH 07/40] Enable OCR by default during development. --- .../testar/visualvalidation/DummyVisualValidator.java | 4 +--- .../visualvalidation/VisualValidationSettings.java | 3 +-- .../nl/ou/testar/visualvalidation/VisualValidator.java | 6 +++--- .../testar/visualvalidation/ocr/OcrResultCallback.java | 10 +++++++++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java index 547ea3959..b3d77e3af 100644 --- a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java @@ -4,9 +4,7 @@ import org.fruit.alayer.AWTCanvas; import org.fruit.alayer.State; -import java.awt.image.BufferedImage; - -public class DummyVisualValidator implements VisualValidationManager{ +public class DummyVisualValidator implements VisualValidationManager { @Override public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java index 692e232d6..0d4a52724 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java @@ -41,7 +41,6 @@ @XmlAccessorType(XmlAccessType.FIELD) public class VisualValidationSettings extends ExtendedSettingBase { public Boolean enabled; - public OcrConfiguration ocrConfiguration; @Override @@ -63,7 +62,7 @@ public String toString() { public static VisualValidationSettings CreateDefault() { VisualValidationSettings DefaultInstance = new VisualValidationSettings(); - DefaultInstance.enabled = false; + DefaultInstance.enabled = true; DefaultInstance.ocrConfiguration = OcrConfiguration.CreateDefault(); return DefaultInstance; } diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 89eaa1ef2..113223db4 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -101,11 +101,11 @@ public void Destroy() { } @Override - public void reportResult(List items) { + public void reportResult(List detectedText) { synchronized (_ocrResultReceived) { - _ocrItems = items; + _ocrItems = detectedText; _ocrResultReceived.notify(); - Logger.log(Level.INFO, TAG, "Received {} result", items.size()); + Logger.log(Level.INFO, TAG, "Received {} result", detectedText.size()); } } } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java index b15c26d2b..73a0169da 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/OcrResultCallback.java @@ -2,6 +2,14 @@ import java.util.List; +/** + * Callback function for sharing the detected text by an OCR engine. + */ public interface OcrResultCallback { - void reportResult(List items); + /** + * Report the detected text to the caller. + * + * @param detectedText The detected text by the OCR engine. + */ + void reportResult(List detectedText); } From 5eb40c2a54b125dcafe53144a4b7c7b3efa284e2 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 12 Dec 2020 12:52:37 +0100 Subject: [PATCH 08/40] Rebase fix. --- testar/src/org/fruit/monkey/DefaultProtocol.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index d59d91bc7..f25259730 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -1541,22 +1541,18 @@ protected State getState(SUT system) throws StateBuildException { */ private void setStateScreenshot(State state) { Shape viewPort = state.get(Tags.Shape, null); - AWTCanvas screenShot = null; + AWTCanvas screenshot = null; if(viewPort != null){ if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)){ - //System.out.println("DEBUG: Using WebDriver specific state shot."); - state.set(Tags.ScreenshotPath, WdProtocolUtil.getStateshot(state)); + screenshot = WdProtocolUtil.getStateshotBinary(state); }else{ - //System.out.println("DEBUG: normal state shot"); - state.set(Tags.ScreenshotPath, ProtocolUtil.getStateshot(state)); + screenshot = ProtocolUtil.getStateshotBinary(state); } - AWTCanvas screenShot = protocolUtil.getStateshotBinary(state); - visualValidationManager.AnalyzeImage(screenShot.image()); String screenshotPath = ScreenshotSerialiser.saveStateshot(state.get(Tags.ConcreteIDCustom, - "NoConcreteIdAvailable"), screenShot); + "NoConcreteIdAvailable"), screenshot); state.set(Tags.ScreenshotPath, screenshotPath); } - visualValidationManager.AnalyzeImage(state, screenShot); + visualValidationManager.AnalyzeImage(state, screenshot); } @Override From 00350f6eac072232b67adbaadb2915bcb36d6bda Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 16 Dec 2020 22:36:16 +0100 Subject: [PATCH 09/40] Initial working text extractor. --- .../testar/visualvalidation/TextElement.java | 37 ++++ .../visualvalidation/VisualValidator.java | 91 ++++++--- .../extractor/DummyExtractor.java | 17 ++ .../extractor/ExpectedElement.java | 19 ++ .../extractor/ExpectedTextCallback.java | 18 ++ .../extractor/ExpectedTextExtractor.java | 173 ++++++++++++++++++ .../extractor/ExtractorFactory.java | 15 ++ .../extractor/TextExtractorInterface.java | 19 ++ .../extractor/WidgetTextConfiguration.java | 47 +++++ .../extractor/WidgetTextSetting.java | 58 ++++++ .../ocr/RecognizedElement.java | 17 +- .../ocr/tesseract/TesseractOcrEngine.java | 17 +- .../src/org/fruit/monkey/DefaultProtocol.java | 4 +- .../settings/ExtendedSettingsFactory.java | 5 + 14 files changed, 493 insertions(+), 44 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/TextElement.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextCallback.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextSetting.java diff --git a/testar/src/nl/ou/testar/visualvalidation/TextElement.java b/testar/src/nl/ou/testar/visualvalidation/TextElement.java new file mode 100644 index 000000000..2b8e41f57 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/TextElement.java @@ -0,0 +1,37 @@ +package nl.ou.testar.visualvalidation; + +public class TextElement { + public final int _x1; + public final int _y1; + public final int _x2; + public final int _y2; + public final String _text; + + /** + * Constructor. + * + * @param x1 The first X coordinate of the text. + * @param y1 The first Y coordinate of the text. + * @param x2 The second X coordinate of the text. + * @param y2 The second Y coordinate of the text. + * @param text The text. + */ + public TextElement(int x1, int y1, int x2, int y2, String text) { + _x1 = x1; + _y1 = y1; + _x2 = x2; + _y2 = y2; + _text = text; + } + + @Override + public String toString() { + return "TextElement{" + + "_x1=" + _x1 + + ", _y1=" + _y1 + + ", _x2=" + _x2 + + ", _y2=" + _y2 + + ", _text='" + _text + '\'' + + '}'; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 113223db4..5e748c246 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -1,5 +1,9 @@ package nl.ou.testar.visualvalidation; +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; +import nl.ou.testar.visualvalidation.extractor.ExpectedTextCallback; +import nl.ou.testar.visualvalidation.extractor.ExtractorFactory; +import nl.ou.testar.visualvalidation.extractor.TextExtractorInterface; import nl.ou.testar.visualvalidation.matcher.VisualMatcher; import nl.ou.testar.visualvalidation.matcher.VisualMatcherFactory; import nl.ou.testar.visualvalidation.ocr.OcrConfiguration; @@ -8,30 +12,41 @@ import nl.ou.testar.visualvalidation.ocr.OcrResultCallback; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.AWTCanvas; import org.fruit.alayer.State; import org.testar.Logger; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; -public class VisualValidator implements VisualValidationManager, OcrResultCallback { - private final OcrEngineInterface ocrEngine; - private final VisualMatcher matcher; +public class VisualValidator implements VisualValidationManager, OcrResultCallback, ExpectedTextCallback { private final String TAG = "VisualValidator"; private int analysisId = 0; - private final Boolean _ocrResultReceived = false; - private final Boolean _expectedTextReceived = false; + + private final VisualMatcher matcher; + + private final OcrEngineInterface _ocrEngine; + private final Object _ocrResultSync = new Object(); + private final AtomicBoolean _ocrResultReceived = new AtomicBoolean(); private List _ocrItems = null; - public VisualValidator(VisualValidationSettings settings) { + private final TextExtractorInterface _extractor; + private final Object _expectedTextSync = new Object(); + private final AtomicBoolean _expectedTextReceived = new AtomicBoolean(); + private List _expectedText = null; + + public VisualValidator(@NonNull VisualValidationSettings settings) { OcrConfiguration ocrConfig = settings.ocrConfiguration; if (ocrConfig.enabled) { - ocrEngine = OcrEngineFactory.createOcrEngine(ocrConfig); + _ocrEngine = OcrEngineFactory.createOcrEngine(ocrConfig); } else { - ocrEngine = null; + _ocrEngine = null; } + _extractor = ExtractorFactory.CreateTextExtractor(); + matcher = VisualMatcherFactory.createDummyMatcher(); } @@ -46,7 +61,7 @@ public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { // Start extracting text, provide callback once finished. extractExpectedText(state); - // Wait for both results. + // Match the expected text with the detected text. matchText(); updateVerdict(state); @@ -56,12 +71,18 @@ public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { private void startNewAnalysis() { analysisId++; + synchronized (_ocrResultSync) { + _ocrResultReceived.set(false); + } + synchronized (_expectedTextSync) { + _expectedTextReceived.set(false); + } Logger.log(Level.INFO, TAG, "Starting new analysis {}", analysisId); } private void parseScreenshot(@Nullable AWTCanvas screenshot) { if (screenshot != null) { - ocrEngine.AnalyzeImage(screenshot.image(), this); + _ocrEngine.AnalyzeImage(screenshot.image(), this); } else { Logger.log(Level.ERROR, TAG, "No screenshot for current state"); @@ -69,19 +90,31 @@ private void parseScreenshot(@Nullable AWTCanvas screenshot) { } private void extractExpectedText(State state) { - Logger.log(Level.INFO, TAG, "Extracting text"); + _extractor.ExtractExpectedText(state, this); } private void matchText() { - synchronized (_ocrResultReceived) { - try { - _ocrResultReceived.wait(); - } catch (InterruptedException e) { - e.printStackTrace(); + waitForResults(); + + Logger.log(Level.INFO, TAG, "Matching {} with {}", _ocrItems, _expectedText); + } + + private void waitForResult(@NonNull AtomicBoolean receivedFlag, Object syncObject) { + if (!receivedFlag.get()) { + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (syncObject) { + try { + syncObject.wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } + } - Logger.log(Level.INFO, TAG, "Matching {} with {}", _ocrItems, "api"); + private void waitForResults() { + waitForResult(_ocrResultReceived, _ocrResultSync); + waitForResult(_expectedTextReceived, _expectedTextSync); } private void updateVerdict(State state) { @@ -95,17 +128,29 @@ private void storeAnalysis() { @Override public void Destroy() { matcher.destroy(); - if (ocrEngine != null) { - ocrEngine.Destroy(); + _extractor.Destroy(); + if (_ocrEngine != null) { + _ocrEngine.Destroy(); } } @Override - public void reportResult(List detectedText) { - synchronized (_ocrResultReceived) { + public void reportResult(@NonNull List detectedText) { + synchronized (_ocrResultSync) { _ocrItems = detectedText; - _ocrResultReceived.notify(); - Logger.log(Level.INFO, TAG, "Received {} result", detectedText.size()); + _ocrResultReceived.set(true); + _ocrResultSync.notifyAll(); + Logger.log(Level.INFO, TAG, "Received {} OCR result", detectedText.size()); + } + } + + @Override + public void ReportExtractedText(@NonNull List expectedText) { + synchronized (_expectedTextSync) { + _expectedText = expectedText; + _expectedTextReceived.set(true); + _expectedTextSync.notifyAll(); + Logger.log(Level.INFO, TAG, "Received {} expected result", expectedText.size()); } } } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java new file mode 100644 index 000000000..698025967 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java @@ -0,0 +1,17 @@ +package nl.ou.testar.visualvalidation.extractor; + +import org.fruit.alayer.State; + +import java.util.ArrayList; + +public class DummyExtractor implements TextExtractorInterface{ + @Override + public void ExtractExpectedText(State state, ExpectedTextCallback callback) { + callback.ReportExtractedText(new ArrayList<>()); + } + + @Override + public void Destroy() { + + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java new file mode 100644 index 000000000..054bd7ccb --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java @@ -0,0 +1,19 @@ +package nl.ou.testar.visualvalidation.extractor; + +import nl.ou.testar.visualvalidation.TextElement; + +public class ExpectedElement extends TextElement { + + /** + * Constructor. + * + * @param x1 The first X coordinate of the text. + * @param y1 The first Y coordinate of the text. + * @param x2 The second X coordinate of the text. + * @param y2 The second Y coordinate of the text. + * @param text The text. + */ + public ExpectedElement(int x1, int y1, int x2, int y2, String text) { + super(x1, y1, x2, y2, text); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextCallback.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextCallback.java new file mode 100644 index 000000000..8e140fd3e --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextCallback.java @@ -0,0 +1,18 @@ +package nl.ou.testar.visualvalidation.extractor; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.fruit.alayer.Widget; + +import java.util.List; + +/** + * Callback function for sharing the expected text for all the {@link Widget}. + */ +public interface ExpectedTextCallback { + /** + * Report the expected text to the caller. + * + * @param expectedText The expected text for all the {@link Widget}. + */ + void ReportExtractedText(@NonNull List expectedText); +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java new file mode 100644 index 000000000..dfbeb60c0 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java @@ -0,0 +1,173 @@ +package nl.ou.testar.visualvalidation.extractor; + + +import org.apache.logging.log4j.Level; +import org.fruit.Util; +import org.fruit.alayer.Roles; +import org.fruit.alayer.Shape; +import org.fruit.alayer.Tag; +import org.fruit.alayer.TagsBase; +import org.fruit.alayer.Widget; +import org.testar.Logger; +import org.testar.settings.ExtendedSettingsFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static org.fruit.alayer.Tags.*; + +public class ExpectedTextExtractor extends Thread implements TextExtractorInterface { + private static final String TAG = "ExpectedTextExtractor"; + + AtomicBoolean running = new AtomicBoolean(true); + + private final Boolean _threadSync = true; + private org.fruit.alayer.State _state = null; + private ExpectedTextCallback _callback = null; + + /** + * A blacklist for {@link Widget}'s which should be ignored based on their {@code Role} because the don't contain + * readable text. Optional when the value (which represents the ancestor path) is set, the {@link Widget} should + * only be ignored when the ancestor path is equal with the {@link Widget} under investigation. + */ + private final Map _blacklist = new HashMap<>(); + + /** + * A lookup table which indicates based on the {@code Role} which {@link Tag} should be used to extract the text. + */ + private final Map _lookupTable = new HashMap<>(); + + /** + * Lookup table to map the name of the name of a {@link Tag}, to the actual {@link Tag}. + * Holds all the available tag's which could hold text (String types, only). The key represents the {@link Tag} in String type, value + */ + @SuppressWarnings("unchecked") + private static final Map> _tag = TagsBase.tagSet().stream() + .filter(tag -> tag.type().equals(String.class)) + .collect(Collectors.toMap(Tag::name, tag -> (Tag) tag)); + + ExpectedTextExtractor() { + WidgetTextConfiguration config = ExtendedSettingsFactory.createWidgetTextConfiguration(); + // Load the extractor configuration into a lookup table for quick access. + config.widget.forEach(it -> { + if (it.ignore) { + _blacklist.put(it.role, it.ancestor); + } else { + _lookupTable.put(it.role, it.tag); + } + }); + + setName(TAG); + start(); + } + + @Override + public void ExtractExpectedText(org.fruit.alayer.State state, ExpectedTextCallback callback) { + synchronized (_threadSync) { + _state = state; + _callback = callback; + Logger.log(Level.TRACE, TAG, "Queue new text extract."); + _threadSync.notifyAll(); + } + } + + @Override + public void run() { + while (running.get()) { + synchronized (_threadSync) { + try { + // Wait until we need to inspect a new image. + _threadSync.wait(); + if (!running.get()) { + break; + } + extractText(); + + } catch (InterruptedException e) { + // Happens if someone interrupts your thread. + Logger.log(Level.INFO, TAG, "Wait interrupted"); + e.printStackTrace(); + } + } + } + } + + private void extractText() { + Objects.requireNonNull(_state); + Objects.requireNonNull(_callback); + + List expectedElements = new ArrayList<>(); + + for (Widget widget : _state) { + String widgetRole = widget.get(Role).name(); + + if (widgetIsIncluded(widget, widgetRole)) { + String text = widget.get(getVisualTextTag(widgetRole), ""); + + if (text != null && !text.isEmpty()) { + Shape dimension = widget.get(Shape, null); + int x1 = dimension != null ? (int) dimension.x() : 0; + int y1 = dimension != null ? (int) dimension.y() : 0; + int x2 = dimension != null ? (int) (dimension.width() + dimension.x()) : 0; + int y2 = dimension != null ? (int) (dimension.height() + dimension.y()) : 0; + + expectedElements.add(new ExpectedElement(x1, y1, x2, y2, text)); + } + } else { + Logger.log(Level.DEBUG, TAG, "Widget {} ignored", widgetRole); + } + } + + _callback.ReportExtractedText(expectedElements); + } + + private boolean widgetIsIncluded(Widget w, String role) { + boolean containsReadableText = true; + try { + String ancestors = _blacklist.get(role); + if (!ancestors.isEmpty()) { + StringBuilder sb = new StringBuilder(); + Util.ancestors(w).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); + + // Check if we should ignore this widget based on its ancestors. + if (sb.toString().equals(ancestors)) { + containsReadableText = false; + } + } + } catch (NullPointerException ignored) { + + } + return containsReadableText; + } + + private Tag getVisualTextTag(String widgetRole) { + // Check if we have specified a custom tag for this widget, if so convert the string identifier into the actual + // Tag. + return _tag.getOrDefault(_lookupTable.getOrDefault(widgetRole, ""), Title); + } + + @Override + public void Destroy() { + stopAndJoinThread(); + + Logger.log(Level.DEBUG, TAG, "Extractor destroyed."); + } + + private void stopAndJoinThread() { + synchronized (_threadSync) { + running.set(false); + _threadSync.notifyAll(); + } + + try { + join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java new file mode 100644 index 000000000..3ae7aa559 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java @@ -0,0 +1,15 @@ +package nl.ou.testar.visualvalidation.extractor; + +public class ExtractorFactory { + public static TextExtractorInterface CreateTextExtractor(){ + return CreateExpectedTextExtractor(); + } + + private static TextExtractorInterface CreateDummyExtractor(){ + return new DummyExtractor(); + } + + private static TextExtractorInterface CreateExpectedTextExtractor(){ + return new ExpectedTextExtractor(); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java b/testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java new file mode 100644 index 000000000..120690650 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java @@ -0,0 +1,19 @@ +package nl.ou.testar.visualvalidation.extractor; + +import org.fruit.alayer.State; +import org.fruit.alayer.Widget; + +public interface TextExtractorInterface { + /** + * Extract the expected text for all the available {@link Widget}'s in the given {@link State}. + * + * @param state The current state of the application under test. + * @param callback Callback function for returning the expected text. + */ + void ExtractExpectedText(State state, ExpectedTextCallback callback); + + /** + * Destroy the text extractor. + */ + void Destroy(); +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java new file mode 100644 index 000000000..6e7399329 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java @@ -0,0 +1,47 @@ +package nl.ou.testar.visualvalidation.extractor; + +import org.apache.commons.lang.ObjectUtils; +import org.testar.settings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class WidgetTextConfiguration extends ExtendedSettingBase { + List widget; + + public static WidgetTextConfiguration CreateDefault() { + WidgetTextConfiguration instance = new WidgetTextConfiguration(); + + WidgetTextSetting scrollbar = WidgetTextSetting.CreateIgnore("UIAScrollBar"); + WidgetTextSetting menuBar = WidgetTextSetting.CreateIgnore("UIAMenuBar"); + WidgetTextSetting textEdit = WidgetTextSetting.CreateExtract("UIAEdit", "UIAValueValue"); + WidgetTextSetting icon = WidgetTextSetting.CreateIgnoreAncestorBased("UIAMenuItem", + Arrays.asList("UIAMenuBar", "UIATitleBar", "UIAWindow", "Process")); + + instance.widget = new ArrayList<>(Arrays.asList(scrollbar, menuBar, textEdit, icon)); + return instance; + } + + @Override + public int compareTo(WidgetTextConfiguration other) { + int result = -1; + + if (widget.size() == other.widget.size()) { + for (int i = 0; i < widget.size(); i++) { + int index = i; + if (other.widget.stream().noneMatch(it -> it.compareTo(widget.get(index)) == 0)){ + return result; + } + } + return 0; + } + return result; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextSetting.java b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextSetting.java new file mode 100644 index 000000000..bb79fa741 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextSetting.java @@ -0,0 +1,58 @@ +package nl.ou.testar.visualvalidation.extractor; + +import org.testar.settings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +import static java.util.Collections.emptyList; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +class WidgetTextSetting extends ExtendedSettingBase { + public static final boolean IGNORE = true; + public static final String ANCESTOR_SEPARATOR = "::"; + + String role; + String tag; + Boolean ignore; + String ancestor; + + private static WidgetTextSetting CreateRaw(String role, Boolean ignore, String tag, List ancestor) { + WidgetTextSetting result = new WidgetTextSetting(); + result.role = role; + result.tag = tag; + result.ignore = ignore; + StringBuilder sb = new StringBuilder(); + ancestor.forEach(it -> sb.append(ANCESTOR_SEPARATOR).append(it)); + result.ancestor = sb.toString(); + return result; + } + + public static WidgetTextSetting CreateIgnoreAncestorBased(String role, List ancestor) { + return CreateRaw(role, IGNORE, "", ancestor); + } + + public static WidgetTextSetting CreateIgnore(String role) { + return CreateRaw(role, IGNORE, "", emptyList()); + } + + public static WidgetTextSetting CreateExtract(String role, String tag) { + return CreateRaw(role, false, tag, emptyList()); + } + + @Override + public int compareTo(WidgetTextSetting other) { + int result = -1; + if (role.contentEquals(other.role) + && (tag.contentEquals(other.tag)) + && (ignore.equals(other.ignore) + && (ancestor.equals(other.ancestor))) + ) { + result = 0; + } + return result; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java index d9a063615..dc5f6b9fa 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java @@ -1,15 +1,12 @@ package nl.ou.testar.visualvalidation.ocr; +import nl.ou.testar.visualvalidation.TextElement; + /** * A discovered text element by an OCR engine. */ -public class RecognizedElement { - final int _x1; - final int _y1; - final int _x2; - final int _y2; - final float _confidence; - final String _text; +public class RecognizedElement extends TextElement { + public final float _confidence; /** * Constructor. @@ -22,12 +19,8 @@ public class RecognizedElement { * @param text The discovered text. */ public RecognizedElement(int x1, int y1, int x2, int y2, float confidence, String text) { - _x1 = x1; - _y1 = y1; - _x2 = x2; - _y2 = y2; + super(x1, y1, x2, y2, text); _confidence = confidence; - _text = text; } @Override diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java index d45b37364..73f64d586 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java @@ -8,6 +8,7 @@ import org.bytedeco.tesseract.ResultIterator; import org.bytedeco.tesseract.TessBaseAPI; import org.bytedeco.tesseract.global.tesseract; +import org.checkerframework.checker.nullness.qual.NonNull; import org.testar.Logger; import org.testar.settings.ExtendedSettingsFactory; @@ -22,13 +23,14 @@ import java.nio.ShortBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; /** * OCR engine implementation dedicated for the Tesseract engine. * Analyzes the given image via {@link #AnalyzeImage} in a separate thread. * Once the engine has finished the caller shall be informed via the given callback. - * + *

* The current implementation can only process one image at the time. This is enforced by the synchronized statements in * {@link #AnalyzeImage} where we set the buffer which is used by the engine and the actual execution of the engine in * the {@link #run}. @@ -66,7 +68,7 @@ public void AnalyzeImage(BufferedImage image, OcrResultCallback callback) { _image = image; _callback = callback; Logger.log(Level.TRACE, TAG, "Queue new image scan."); - _scanSync.notify(); + _scanSync.notifyAll(); } } @@ -92,6 +94,9 @@ public void run() { } private void recognizeText() { + Objects.requireNonNull(_image); + Objects.requireNonNull(_callback); + List recognizedWords = new ArrayList<>(); loadImageIntoEngine(_image); @@ -104,18 +109,14 @@ private void recognizeText() { do { recognizedWords.add(TesseractResult.Extract(recognizedElement, level)); } while (recognizedElement.Next(level)); - - recognizedWords.forEach(ocrWord -> Logger.log(Level.DEBUG, TAG, "Found {}", ocrWord)); } } _engine.Clear(); // Notify the callback with the discovered words. _callback.reportResult(recognizedWords); - - Logger.log(Level.DEBUG, TAG, "Finished image scan found {} elements.", recognizedWords.size()); } - private void loadImageIntoEngine(BufferedImage image) { + private void loadImageIntoEngine(@NonNull BufferedImage image) { DataBuffer dataBuffer = image.getData().getDataBuffer(); ByteBuffer byteBuffer; @@ -157,7 +158,7 @@ public void Destroy() { private void stopAndJoinThread() { synchronized (_scanSync) { running.set(false); - _scanSync.notify(); + _scanSync.notifyAll(); } try { diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index f25259730..c8e2517c4 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -1888,7 +1888,9 @@ private void closeTestarTestSession(){ GlobalScreen.unregisterNativeHook(); } } - visualValidationManager.Destroy(); + if (visualValidationManager != null) { + visualValidationManager.Destroy(); + } } catch(Exception e) { e.printStackTrace(); } diff --git a/testar/src/org/testar/settings/ExtendedSettingsFactory.java b/testar/src/org/testar/settings/ExtendedSettingsFactory.java index c72be9687..cb75f0847 100644 --- a/testar/src/org/testar/settings/ExtendedSettingsFactory.java +++ b/testar/src/org/testar/settings/ExtendedSettingsFactory.java @@ -31,6 +31,7 @@ package org.testar.settings; import nl.ou.testar.visualvalidation.VisualValidationSettings; +import nl.ou.testar.visualvalidation.extractor.WidgetTextConfiguration; import nl.ou.testar.visualvalidation.ocr.tesseract.TesseractSettings; import java.util.ArrayList; @@ -89,4 +90,8 @@ public static ExampleSetting createTestSetting() { public static TesseractSettings createTesseractSetting() { return createSettings(TesseractSettings.class, TesseractSettings::CreateDefault); } + + public static WidgetTextConfiguration createWidgetTextConfiguration() { + return createSettings(WidgetTextConfiguration.class, WidgetTextConfiguration::CreateDefault); + } } From edd523fcf98d40103553c80101ed13bbd40bab29 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Tue, 29 Dec 2020 15:15:43 +0100 Subject: [PATCH 10/40] Added workaround for null pointer. Investigation required why the loop is triggered before the event has been queued. --- .../visualvalidation/extractor/ExpectedTextExtractor.java | 8 +++++--- .../extractor/WidgetTextConfiguration.java | 3 +-- .../ocr/tesseract/TesseractOcrEngine.java | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java index dfbeb60c0..abab71431 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java @@ -15,11 +15,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.fruit.alayer.Tags.*; +import static org.fruit.alayer.webdriver.enums.WdTags.WebTextContent; public class ExpectedTextExtractor extends Thread implements TextExtractorInterface { private static final String TAG = "ExpectedTextExtractor"; @@ -98,8 +98,10 @@ public void run() { } private void extractText() { - Objects.requireNonNull(_state); - Objects.requireNonNull(_callback); + if (_state == null || _callback == null) { + Logger.log(Level.ERROR, TAG, "Should not try to extract text on empty state/callback"); + return; + } List expectedElements = new ArrayList<>(); diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java index 6e7399329..0dc291c84 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java @@ -1,6 +1,5 @@ package nl.ou.testar.visualvalidation.extractor; -import org.apache.commons.lang.ObjectUtils; import org.testar.settings.ExtendedSettingBase; import javax.xml.bind.annotation.XmlAccessType; @@ -36,7 +35,7 @@ public int compareTo(WidgetTextConfiguration other) { if (widget.size() == other.widget.size()) { for (int i = 0; i < widget.size(); i++) { int index = i; - if (other.widget.stream().noneMatch(it -> it.compareTo(widget.get(index)) == 0)){ + if (other.widget.stream().noneMatch(it -> it.compareTo(widget.get(index)) == 0)) { return result; } } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java index 73f64d586..3af3520f7 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java @@ -94,9 +94,10 @@ public void run() { } private void recognizeText() { - Objects.requireNonNull(_image); - Objects.requireNonNull(_callback); - + if (_image == null || _callback == null) { + Logger.log(Level.ERROR, TAG, "Should not try to detect text on empty image/callback"); + return; + } List recognizedWords = new ArrayList<>(); loadImageIntoEngine(_image); From 77dd386dbf9747b3175b656d47347e68850dd129 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Tue, 29 Dec 2020 15:16:31 +0100 Subject: [PATCH 11/40] Added workaround to support webdriver: default tag based on the Tag name. --- .../visualvalidation/extractor/ExpectedTextExtractor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java index abab71431..e57d37fb3 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java @@ -150,7 +150,9 @@ private boolean widgetIsIncluded(Widget w, String role) { private Tag getVisualTextTag(String widgetRole) { // Check if we have specified a custom tag for this widget, if so convert the string identifier into the actual // Tag. - return _tag.getOrDefault(_lookupTable.getOrDefault(widgetRole, ""), Title); + // TODO TM: Should inject this based on the selected protocol? + Tag defaultTag = widgetRole.contains("Wd") ? WebTextContent : Title; + return _tag.getOrDefault(_lookupTable.getOrDefault(widgetRole, ""), defaultTag); } @Override From 14d645424a11c686554150cb40a990a873ef582e Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 16 Jan 2021 16:47:46 +0100 Subject: [PATCH 12/40] Replaced raw points for TextElements with rectangle class. --- .../testar/visualvalidation/TextElement.java | 26 +++----- .../extractor/ExpectedElement.java | 13 ++-- .../extractor/ExpectedTextExtractor.java | 66 +++++++++++++------ .../extractor/WidgetTextConfiguration.java | 5 +- .../ocr/RecognizedElement.java | 16 ++--- .../ocr/tesseract/TesseractResult.java | 25 ++++--- 6 files changed, 85 insertions(+), 66 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/TextElement.java b/testar/src/nl/ou/testar/visualvalidation/TextElement.java index 2b8e41f57..c007ab1d6 100644 --- a/testar/src/nl/ou/testar/visualvalidation/TextElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/TextElement.java @@ -1,36 +1,26 @@ package nl.ou.testar.visualvalidation; +import java.awt.Rectangle; + public class TextElement { - public final int _x1; - public final int _y1; - public final int _x2; - public final int _y2; + public final Rectangle _location; public final String _text; /** * Constructor. * - * @param x1 The first X coordinate of the text. - * @param y1 The first Y coordinate of the text. - * @param x2 The second X coordinate of the text. - * @param y2 The second Y coordinate of the text. - * @param text The text. + * @param location The relative location of the text inside the application. + * @param text The text. */ - public TextElement(int x1, int y1, int x2, int y2, String text) { - _x1 = x1; - _y1 = y1; - _x2 = x2; - _y2 = y2; + public TextElement(Rectangle location, String text) { + _location = location; _text = text; } @Override public String toString() { return "TextElement{" + - "_x1=" + _x1 + - ", _y1=" + _y1 + - ", _x2=" + _x2 + - ", _y2=" + _y2 + + "_location=" + _location + ", _text='" + _text + '\'' + '}'; } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java index 054bd7ccb..2d3b31795 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java @@ -2,18 +2,17 @@ import nl.ou.testar.visualvalidation.TextElement; +import java.awt.Rectangle; + public class ExpectedElement extends TextElement { /** * Constructor. * - * @param x1 The first X coordinate of the text. - * @param y1 The first Y coordinate of the text. - * @param x2 The second X coordinate of the text. - * @param y2 The second Y coordinate of the text. - * @param text The text. + * @param location The relative location of the text inside the application. + * @param text The text. */ - public ExpectedElement(int x1, int y1, int x2, int y2, String text) { - super(x1, y1, x2, y2, text); + public ExpectedElement(Rectangle location, String text) { + super(location, text); } } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java index e57d37fb3..fb533d194 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java @@ -2,19 +2,23 @@ import org.apache.logging.log4j.Level; +import org.fruit.Environment; import org.fruit.Util; import org.fruit.alayer.Roles; import org.fruit.alayer.Shape; import org.fruit.alayer.Tag; +import org.fruit.alayer.Tags; import org.fruit.alayer.TagsBase; import org.fruit.alayer.Widget; import org.testar.Logger; import org.testar.settings.ExtendedSettingsFactory; +import java.awt.Rectangle; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -23,6 +27,7 @@ public class ExpectedTextExtractor extends Thread implements TextExtractorInterface { private static final String TAG = "ExpectedTextExtractor"; + private final String rootElementPath = "[0]"; AtomicBoolean running = new AtomicBoolean(true); @@ -97,14 +102,36 @@ public void run() { } } + static Rectangle getLocation(Widget widget) { + Shape dimension = widget.get(Shape, null); + + int x = dimension != null ? (int) dimension.x() : 0; + int y = dimension != null ? (int) dimension.y() : 0; + int width = dimension != null ? (int) dimension.width() : 0; + int height = dimension != null ? (int) dimension.height() : 0; + + return new Rectangle(x, y, width, height); + } + private void extractText() { if (_state == null || _callback == null) { Logger.log(Level.ERROR, TAG, "Should not try to extract text on empty state/callback"); return; } - List expectedElements = new ArrayList<>(); + // Acquire the absolute location of the SUT on the screen. + Rectangle applicationPosition = null; + for (Widget widget : _state) { + if (widget.get(Path).contentEquals(rootElementPath)) { + applicationPosition = getLocation(widget); + break; + } + } + Objects.requireNonNull(applicationPosition); + + double displayScale = Environment.getInstance().getDisplayScale(_state.child(0).get(Tags.HWND, (long) 0)); + List expectedElements = new ArrayList<>(); for (Widget widget : _state) { String widgetRole = widget.get(Role).name(); @@ -112,13 +139,13 @@ private void extractText() { String text = widget.get(getVisualTextTag(widgetRole), ""); if (text != null && !text.isEmpty()) { - Shape dimension = widget.get(Shape, null); - int x1 = dimension != null ? (int) dimension.x() : 0; - int y1 = dimension != null ? (int) dimension.y() : 0; - int x2 = dimension != null ? (int) (dimension.width() + dimension.x()) : 0; - int y2 = dimension != null ? (int) (dimension.height() + dimension.y()) : 0; - - expectedElements.add(new ExpectedElement(x1, y1, x2, y2, text)); + Rectangle absoluteLocation = getLocation(widget); + Rectangle relativeLocation = new Rectangle( + (int) ((absoluteLocation.x - applicationPosition.x) * displayScale), + (int) ((absoluteLocation.y - applicationPosition.y) * displayScale), + absoluteLocation.width, + absoluteLocation.height); + expectedElements.add(new ExpectedElement(relativeLocation, text)); } } else { Logger.log(Level.DEBUG, TAG, "Widget {} ignored", widgetRole); @@ -130,19 +157,20 @@ private void extractText() { private boolean widgetIsIncluded(Widget w, String role) { boolean containsReadableText = true; - try { - String ancestors = _blacklist.get(role); - if (!ancestors.isEmpty()) { - StringBuilder sb = new StringBuilder(); - Util.ancestors(w).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); - - // Check if we should ignore this widget based on its ancestors. - if (sb.toString().equals(ancestors)) { - containsReadableText = false; + if (_blacklist.containsKey(role)) { + containsReadableText = false; + try { + String ancestors = _blacklist.get(role); + if (!ancestors.isEmpty()) { + StringBuilder sb = new StringBuilder(); + Util.ancestors(w).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); + + // Check if we should ignore this widget based on its ancestors. + containsReadableText = !sb.toString().equals(ancestors); } - } - } catch (NullPointerException ignored) { + } catch (NullPointerException ignored) { + } } return containsReadableText; } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java index 0dc291c84..d01ceecc2 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java @@ -18,13 +18,14 @@ public class WidgetTextConfiguration extends ExtendedSettingBase(Arrays.asList(scrollbar, menuBar, textEdit, icon)); + instance.widget = new ArrayList<>(Arrays.asList(scrollBar, statusBar, menuBar, textEdit, icon)); return instance; } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java index dc5f6b9fa..03b208e52 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java @@ -2,6 +2,8 @@ import nl.ou.testar.visualvalidation.TextElement; +import java.awt.Rectangle; + /** * A discovered text element by an OCR engine. */ @@ -11,25 +13,19 @@ public class RecognizedElement extends TextElement { /** * Constructor. * - * @param x1 The first X coordinate of the discovered text. - * @param y1 The first Y coordinate of the discovered text. - * @param x2 The second X coordinate of the discovered text. - * @param y2 The second Y coordinate of the discovered text. + * @param location The relative location of the text inside the application. * @param confidence The confidence level of the discovered text. * @param text The discovered text. */ - public RecognizedElement(int x1, int y1, int x2, int y2, float confidence, String text) { - super(x1, y1, x2, y2, text); + public RecognizedElement(Rectangle location, float confidence, String text) { + super(location, text); _confidence = confidence; } @Override public String toString() { return "RecognizedElement{" + - "_x1=" + _x1 + - ", _y1=" + _y1 + - ", _x2=" + _x2 + - ", _y2=" + _y2 + + "_location=" + _location + ", _confidence=" + _confidence + ", _text='" + _text + '\'' + '}'; diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java index 843e77d34..57b3f5b7c 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java @@ -5,6 +5,7 @@ import org.bytedeco.javacpp.IntPointer; import org.bytedeco.tesseract.ResultIterator; +import java.awt.Rectangle; import java.util.function.Supplier; /** @@ -27,22 +28,26 @@ static RecognizedElement Extract(ResultIterator recognizedElement, int granulari String recognizedText = ocrResult.getString().trim(); float confidence = recognizedElement.Confidence(granularity); - IntPointer x1 = intPointerSupplier.get(); - IntPointer y1 = intPointerSupplier.get(); - IntPointer x2 = intPointerSupplier.get(); - IntPointer y2 = intPointerSupplier.get(); - boolean foundRectangle = recognizedElement.BoundingBox(granularity, x1, y1, x2, y2); + IntPointer left = intPointerSupplier.get(); + IntPointer top = intPointerSupplier.get(); + IntPointer right = intPointerSupplier.get(); + IntPointer bottom = intPointerSupplier.get(); + boolean foundRectangle = recognizedElement.BoundingBox(granularity, left, top, right, bottom); if (!foundRectangle) { throw new IllegalArgumentException("Could not find any rectangle for this element"); } - RecognizedElement result = new RecognizedElement(x1.get(), y1.get(), x2.get(), y2.get(), confidence, recognizedText); + // Upper left coordinate = 0,0 + int width = right.get() - left.get(); + int height = bottom.get() - top.get(); + Rectangle location = new Rectangle(left.get(), top.get(), width, height); + RecognizedElement result = new RecognizedElement(location, confidence, recognizedText); - x1.deallocate(); - y1.deallocate(); - x2.deallocate(); - y2.deallocate(); + left.deallocate(); + top.deallocate(); + right.deallocate(); + bottom.deallocate(); ocrResult.deallocate(); return result; From 315f91c7ba547f481d0928d0bb951e4008c0e9e6 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 16 Jan 2021 16:53:09 +0100 Subject: [PATCH 13/40] WIP: Initial location matcher implementation. --- .../visualvalidation/VisualValidator.java | 32 +++++++- .../matcher/LocationMatcher.java | 80 +++++++++++++++++++ .../visualvalidation/matcher/Match.java | 32 ++++++++ .../matcher/MatchLocation.java | 21 +++++ .../matcher/MatcherResult.java | 35 ++++++++ .../matcher/VisualDummyMatcher.java | 10 +++ .../matcher/VisualMatcher.java | 7 ++ .../matcher/VisualMatcherFactory.java | 8 +- 8 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/Match.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 5e748c246..27b947b14 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -4,6 +4,7 @@ import nl.ou.testar.visualvalidation.extractor.ExpectedTextCallback; import nl.ou.testar.visualvalidation.extractor.ExtractorFactory; import nl.ou.testar.visualvalidation.extractor.TextExtractorInterface; +import nl.ou.testar.visualvalidation.matcher.MatcherResult; import nl.ou.testar.visualvalidation.matcher.VisualMatcher; import nl.ou.testar.visualvalidation.matcher.VisualMatcherFactory; import nl.ou.testar.visualvalidation.ocr.OcrConfiguration; @@ -15,7 +16,14 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.AWTCanvas; +import org.fruit.alayer.AbsolutePosition; +import org.fruit.alayer.Color; +import org.fruit.alayer.FillPattern; +import org.fruit.alayer.Pen; import org.fruit.alayer.State; +import org.fruit.alayer.StrokePattern; +import org.fruit.alayer.Verdict; +import org.fruit.alayer.visualizers.TextVisualizer; import org.testar.Logger; import java.util.List; @@ -25,7 +33,8 @@ public class VisualValidator implements VisualValidationManager, OcrResultCallba private final String TAG = "VisualValidator"; private int analysisId = 0; - private final VisualMatcher matcher; + private final VisualMatcher _matcher; + private MatcherResult _matcherResult = null; private final OcrEngineInterface _ocrEngine; private final Object _ocrResultSync = new Object(); @@ -37,6 +46,9 @@ public class VisualValidator implements VisualValidationManager, OcrResultCallba private final AtomicBoolean _expectedTextReceived = new AtomicBoolean(); private List _expectedText = null; + protected final static Pen RedPen = Pen.newPen().setColor(Color.Red). + setFillPattern(FillPattern.None).setStrokePattern(StrokePattern.Solid).build(); + public VisualValidator(@NonNull VisualValidationSettings settings) { OcrConfiguration ocrConfig = settings.ocrConfiguration; if (ocrConfig.enabled) { @@ -47,7 +59,7 @@ public VisualValidator(@NonNull VisualValidationSettings settings) { _extractor = ExtractorFactory.CreateTextExtractor(); - matcher = VisualMatcherFactory.createDummyMatcher(); + _matcher = VisualMatcherFactory.createLocationMatcher(); } @Override @@ -77,6 +89,7 @@ private void startNewAnalysis() { synchronized (_expectedTextSync) { _expectedTextReceived.set(false); } + _matcherResult = null; Logger.log(Level.INFO, TAG, "Starting new analysis {}", analysisId); } @@ -96,7 +109,7 @@ private void extractExpectedText(State state) { private void matchText() { waitForResults(); - Logger.log(Level.INFO, TAG, "Matching {} with {}", _ocrItems, _expectedText); + _matcherResult = _matcher.Match(_ocrItems, _expectedText); } private void waitForResult(@NonNull AtomicBoolean receivedFlag, Object syncObject) { @@ -118,6 +131,17 @@ private void waitForResults() { } private void updateVerdict(State state) { + + if (_matcherResult != null) { + // Analysis the raw result and create a verdict. + _matcherResult.getMatches(); + + Verdict result = new Verdict(Verdict.SEVERITY_WARNING, "Not all texts has been recognized", + new TextVisualizer(new AbsolutePosition(10, 10), "->", RedPen)); + + } else { + // Set verdict to failure we should have a matcher result as minimal input. + } Logger.log(Level.INFO, TAG, "Updating verdict {}"); } @@ -127,7 +151,7 @@ private void storeAnalysis() { @Override public void Destroy() { - matcher.destroy(); + _matcher.destroy(); _extractor.Destroy(); if (_ocrEngine != null) { _ocrEngine.Destroy(); diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java new file mode 100644 index 000000000..11a9ad3d8 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java @@ -0,0 +1,80 @@ +package nl.ou.testar.visualvalidation.matcher; + +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; +import org.apache.logging.log4j.Level; +import org.testar.Logger; + +import java.awt.Rectangle; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class LocationMatcher implements VisualMatcher { + static final String TAG = "Matcher"; + + @Override + public MatcherResult Match(List ocrResult, List expectedText) { + MatcherResult result = new MatcherResult(); + List _ocrResult = new ArrayList<>(ocrResult); + + expectedText.forEach(expectedElement -> { + final Match[] match = {null}; + Set removeRecognizedList = new HashSet<>(); + _ocrResult.forEach(it -> { + // TODO TM: Lookup margin via extended settings framework + int margin = 0; + Rectangle areaOfInterest = it._location; + areaOfInterest.setBounds( + areaOfInterest.x - margin, + areaOfInterest.y - margin, + areaOfInterest.width + (2 * margin), + areaOfInterest.height + (2 * margin) + ); + + // If the recognized element lay inside the area of interest. + if (areaOfInterest.intersects(expectedElement._location)) { + // Prepare to make a match. + if (match[0] == null) { + match[0] = new Match(expectedElement, margin); + } + + // TODO TM: optimizing this is part of thesis, focus on equality without taking casing into account. + if (expectedElement._text.compareToIgnoreCase(it._text) == 0) { + removeRecognizedList.add(it); + // TODO TM: If all expected items are found should we break? Or what if we find other items that intersect?! + } + match[0].addRecognizedElement(it); + } + }); + + if (match[0] != null) { + result.addMatch(match[0]); + + // TODO TM: Find solution to enable this to speed up the matcher. Currently when the "title bar" is + // being matched it uses the entire window as area of interest so all OCR items are marked as + // interesting, but if a text in title bar is also found in a different location it will be removed + // for an invalid reason. + // Remove the items from the recognized list. + // removeRecognizedList.forEach(_ocrResult::remove); + } + }); + + Set removeRecognizedList = new HashSet<>(); + result.getMatches().forEach(it -> removeRecognizedList.addAll(it.recognizedElements)); + // Remove the elements which have been matched with expected elements. + removeRecognizedList.forEach(_ocrResult::remove); + + // The remainders in OCR result can be added to no match? + _ocrResult.forEach(result::addNoMatch); + + Logger.log(Level.INFO, TAG, "Result {}", result); + return result; + } + + @Override + public void destroy() { + + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/Match.java b/testar/src/nl/ou/testar/visualvalidation/matcher/Match.java new file mode 100644 index 000000000..452867586 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/Match.java @@ -0,0 +1,32 @@ +package nl.ou.testar.visualvalidation.matcher; + +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.HashSet; +import java.util.Set; + +public class Match { + final public MatchLocation location; + final public ExpectedElement expectedElement; + + public Set recognizedElements = new HashSet<>(); + + public Match(ExpectedElement expectedElement, int margin) { + this.location = new MatchLocation(margin, expectedElement._location); + this.expectedElement = expectedElement; + } + + public void addRecognizedElement(RecognizedElement element) { + recognizedElements.add(element); + } + + @Override + public String toString() { + return "Match{" + + "location=" + location + + ", expectedElement=" + expectedElement + + ", recognizedElements=" + recognizedElements + + '}'; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java new file mode 100644 index 000000000..3217ad10b --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java @@ -0,0 +1,21 @@ +package nl.ou.testar.visualvalidation.matcher; + +import java.awt.Rectangle; + +public class MatchLocation { + final public int margin; + final public Rectangle location; + + public MatchLocation(int margin, Rectangle location) { + this.margin = margin; + this.location = location; + } + + @Override + public String toString() { + return "MatchLocation{" + + "margin=" + margin + + ", location=" + location + + '}'; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java new file mode 100644 index 000000000..56ac4db63 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java @@ -0,0 +1,35 @@ +package nl.ou.testar.visualvalidation.matcher; + +import nl.ou.testar.visualvalidation.TextElement; + +import java.util.HashSet; +import java.util.Set; + +public class MatcherResult { + private final Set noMatches = new HashSet<>(); + private final Set matches = new HashSet<>(); + + public void addMatch(Match match) { + matches.add(match); + } + + public void addNoMatch(TextElement element) { + noMatches.add(element); + } + + public Set getNoMatches() { + return noMatches; + } + + public Set getMatches() { + return matches; + } + + @Override + public String toString() { + return "MatcherResult{" + + "noMatches=" + noMatches + + ", matches=" + matches + + '}'; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java index 3fa204132..ee8da8efe 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java @@ -1,6 +1,16 @@ package nl.ou.testar.visualvalidation.matcher; +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.List; + public class VisualDummyMatcher implements VisualMatcher { + @Override + public MatcherResult Match(List ocrResult, List expectedText) { + return new MatcherResult(); + } + @Override public void destroy() { diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java index 5291c0380..811f958ca 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java @@ -1,5 +1,12 @@ package nl.ou.testar.visualvalidation.matcher; +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.List; + public interface VisualMatcher { + MatcherResult Match(List ocrResult, List expectedText); + void destroy(); } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java index 11963b59b..b1224ad66 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java @@ -1,5 +1,11 @@ package nl.ou.testar.visualvalidation.matcher; public class VisualMatcherFactory { - public static VisualMatcher createDummyMatcher() { return new VisualDummyMatcher(); } + public static VisualMatcher createDummyMatcher() { + return new VisualDummyMatcher(); + } + + public static VisualMatcher createLocationMatcher() { + return new LocationMatcher(); + } } From edd8a593c565272cd69a4314b2eda71136425b54 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Thu, 25 Feb 2021 20:42:01 +0100 Subject: [PATCH 14/40] Fixed filtering for a role with different ancestors --- .../extractor/ExpectedTextExtractor.java | 16 +++++++++++----- .../extractor/WidgetTextConfiguration.java | 6 +++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java index fb533d194..70e28c906 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java @@ -15,12 +15,14 @@ import java.awt.Rectangle; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.fruit.alayer.Tags.*; import static org.fruit.alayer.webdriver.enums.WdTags.WebTextContent; @@ -40,7 +42,7 @@ public class ExpectedTextExtractor extends Thread implements TextExtractorInterf * readable text. Optional when the value (which represents the ancestor path) is set, the {@link Widget} should * only be ignored when the ancestor path is equal with the {@link Widget} under investigation. */ - private final Map _blacklist = new HashMap<>(); + private final Map> _blacklist = new HashMap<>(); /** * A lookup table which indicates based on the {@code Role} which {@link Tag} should be used to extract the text. @@ -61,7 +63,10 @@ public class ExpectedTextExtractor extends Thread implements TextExtractorInterf // Load the extractor configuration into a lookup table for quick access. config.widget.forEach(it -> { if (it.ignore) { - _blacklist.put(it.role, it.ancestor); + List ancestor = it.ancestor.isEmpty() ? + Collections.emptyList() : Collections.singletonList(it.ancestor); + _blacklist.merge(it.role, ancestor, (list1, list2) -> + Stream.concat(list1.stream(), list2.stream()).collect(Collectors.toList())); } else { _lookupTable.put(it.role, it.tag); } @@ -160,13 +165,14 @@ private boolean widgetIsIncluded(Widget w, String role) { if (_blacklist.containsKey(role)) { containsReadableText = false; try { - String ancestors = _blacklist.get(role); - if (!ancestors.isEmpty()) { + List blacklistedAncestors = _blacklist.get(role); + if (!blacklistedAncestors.isEmpty()) { StringBuilder sb = new StringBuilder(); Util.ancestors(w).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); // Check if we should ignore this widget based on its ancestors. - containsReadableText = !sb.toString().equals(ancestors); + String ancestors = sb.toString(); + containsReadableText = blacklistedAncestors.stream().noneMatch(ancestors::equals); } } catch (NullPointerException ignored) { diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java index d01ceecc2..529cc1b8c 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java @@ -24,8 +24,12 @@ public static WidgetTextConfiguration CreateDefault() { WidgetTextSetting textEdit = WidgetTextSetting.CreateExtract("UIAEdit", "UIAValueValue"); WidgetTextSetting icon = WidgetTextSetting.CreateIgnoreAncestorBased("UIAMenuItem", Arrays.asList("UIAMenuBar", "UIATitleBar", "UIAWindow", "Process")); + WidgetTextSetting toolBarButtons = WidgetTextSetting.CreateIgnoreAncestorBased("UIAButton", + Arrays.asList("UIATitleBar", "UIAWindow", "Process")); + WidgetTextSetting scrollBarButtons = WidgetTextSetting.CreateIgnoreAncestorBased("UIAButton", + Arrays.asList("UIAScrollBar", "UIAEdit", "UIAWindow", "Process")); - instance.widget = new ArrayList<>(Arrays.asList(scrollBar, statusBar, menuBar, textEdit, icon)); + instance.widget = new ArrayList<>(Arrays.asList(scrollBar, statusBar, menuBar, textEdit, icon, toolBarButtons, scrollBarButtons)); return instance; } From 9d7540b976d05a52ab3bf9f5e4a42c9fa539063b Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 27 Feb 2021 11:24:17 +0100 Subject: [PATCH 15/40] Fixed dimension calculation for expected text. --- .../visualvalidation/extractor/ExpectedTextExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java index 70e28c906..5c4d8e3b4 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java @@ -148,8 +148,8 @@ private void extractText() { Rectangle relativeLocation = new Rectangle( (int) ((absoluteLocation.x - applicationPosition.x) * displayScale), (int) ((absoluteLocation.y - applicationPosition.y) * displayScale), - absoluteLocation.width, - absoluteLocation.height); + (int) (absoluteLocation.width * displayScale), + (int) (absoluteLocation.height * displayScale)); expectedElements.add(new ExpectedElement(relativeLocation, text)); } } else { From 08d41731cb749a0fc8242d17dc736cf234317f92 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 27 Feb 2021 11:30:25 +0100 Subject: [PATCH 16/40] Sort and link the expected text based on their area size. Sorting is done from small to big. By doing so we can speed up the matching algorithm. Elements which have been linked to an expected text element based on their location are removed from the list with the remaining elements which we need to match. --- .../matcher/LocationMatcher.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java index 11a9ad3d8..f1a006455 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Level; import org.testar.Logger; +import java.awt.Dimension; import java.awt.Rectangle; import java.util.ArrayList; import java.util.HashSet; @@ -19,6 +20,16 @@ public MatcherResult Match(List ocrResult, List _ocrResult = new ArrayList<>(ocrResult); + // We first match them based on their location, if they intersect we mark them as match. + // Because there are windows and frames included as well we need to make sure that we don't assign them to + // these elements before we can assign them to the actual widget which is presented on the panel/window. + expectedText.sort((element1, element2) -> { + // Sort based on the area size of the element, from small to big + Dimension element1Size = element1._location.getSize(); + Dimension element2Size = element2._location.getSize(); + return ((element1Size.width * element1Size.height) - (element2Size.width * element2Size.height)); + }); + expectedText.forEach(expectedElement -> { final Match[] match = {null}; Set removeRecognizedList = new HashSet<>(); @@ -52,12 +63,8 @@ public MatcherResult Match(List ocrResult, List Date: Sun, 11 Apr 2021 14:10:14 +0200 Subject: [PATCH 17/40] WIP match content --- .../matcher/LocationMatcher.java | 126 ++++++++++++++++-- .../matcher/MatcherResult.java | 4 +- 2 files changed, 120 insertions(+), 10 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java index f1a006455..a90d5c555 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java @@ -3,14 +3,17 @@ import nl.ou.testar.visualvalidation.extractor.ExpectedElement; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.apache.logging.log4j.Level; +import org.fruit.Pair; import org.testar.Logger; import java.awt.Dimension; import java.awt.Rectangle; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; public class LocationMatcher implements VisualMatcher { static final String TAG = "Matcher"; @@ -50,21 +53,21 @@ public MatcherResult Match(List ocrResult, List ocrResult, List sorted = match.recognizedElements.stream() + .sorted(Comparator.comparingInt(o -> o._location.x)) + .collect(Collectors.toList()); + + // Create list of Found chars and matched FLAG + List> unMatched = sorted.stream().flatMap( + recognizedElement -> recognizedElement._text.chars().boxed().map(it -> Character.toChars(it)[0])) + .map(it -> Pair.from(it, false)) + .collect(Collectors.toList()); + + + // Try to make the longest possible match + // For each char that matches the expected text mark as matched and move one ahead. + + // If not equal, could it be a case mismatch? + // if they are not equal at all + + + List result = new ArrayList<>(); + int unMatchedSize = unMatched.size(); + int indexCounter = 0; + for (int i = 0; i < expectedText.length(); i++) { + Character actual = expectedText.charAt(i); + Test charRes = new Test(actual); + + // Iterate over the + for (int k = indexCounter; k < unMatchedSize; k++) { + Pair item = unMatched.get(k); + Character found = item.left(); + // Try to match the actual char case sensitive: + if (found == actual && !item.right()) { + charRes.found = found; + unMatched.set(k, Pair.from(found, true)); + charRes.matchRes = RES.MATCHED; + // We have found a match inside the unMatched List move one position. + indexCounter = k + 1; + break; + } + + if (Character.isLetter(actual)) { + // Try to match the actual char none sensitive: + int CASING = 32; + if (Math.abs(found.compareTo(actual)) == CASING && !item.right()) { + charRes.found = found; + unMatched.set(k, Pair.from(found, true)); + charRes.matchRes = RES.CASE_MISMATCH; + // We have found a match inside the unMatched List move one position. + indexCounter = k + 1; + break; + } + } + } + + result.add(i, charRes); + } + + // Try to auto correct the missing whitespace + // Skip the first and last since we can't match left and right element + // TODO TM: Should we try to convert all the white spaces + int almostLast = result.size() - 1; + for (int i = 1; i < almostLast; i++) { + + Test res = result.get(i); + if (res.matchRes == RES.NO_MATCH && Character.isWhitespace(res.expected)){ + // Found a candidate, try to fix it + if (result.get(i-1).matchRes != RES.NO_MATCH && result.get(i+1).matchRes != RES.NO_MATCH ){ + // Surroundings are matched this must be a missing whitespace not detected by OCR + res.matchRes = RES.WHITESPACE_CORRECTED; + result.set(i, res); + Logger.log(Level.INFO, TAG, "Auto corrected whitespace in between expected text"); + } + } + } + + result.forEach(i -> Logger.log(Level.INFO, TAG, "{}", i)); + } } + +//TODO see how to deal with reconginized elements which are placed/assinged to incorrect text fields diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java index 56ac4db63..6f4fbb1e2 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java @@ -28,8 +28,8 @@ public Set getMatches() { @Override public String toString() { return "MatcherResult{" + - "noMatches=" + noMatches + - ", matches=" + matches + + "noMatches(" + noMatches.size() + ")=" + noMatches + + ", matches(" + matches.size() + ")=" + matches + '}'; } } From 1af3d98ea6088a8c7ae55b9473823a3cbc2ae52d Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sun, 11 Apr 2021 21:25:02 +0200 Subject: [PATCH 18/40] By setting HWND for root element we can request display scale of both webdriver and desktop protocol. --- .../visualvalidation/extractor/ExpectedTextExtractor.java | 3 +-- windows/src/org/fruit/alayer/windows/StateFetcher.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java index 5c4d8e3b4..965f62fbe 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java @@ -134,8 +134,7 @@ private void extractText() { } Objects.requireNonNull(applicationPosition); - - double displayScale = Environment.getInstance().getDisplayScale(_state.child(0).get(Tags.HWND, (long) 0)); + double displayScale = Environment.getInstance().getDisplayScale(_state.get(Tags.HWND, (long) 0)); List expectedElements = new ArrayList<>(); for (Widget widget : _state) { String widgetRole = widget.get(Role).name(); diff --git a/windows/src/org/fruit/alayer/windows/StateFetcher.java b/windows/src/org/fruit/alayer/windows/StateFetcher.java index 759159565..ae7f3f410 100644 --- a/windows/src/org/fruit/alayer/windows/StateFetcher.java +++ b/windows/src/org/fruit/alayer/windows/StateFetcher.java @@ -104,7 +104,7 @@ public UIAState call() throws Exception { UIAState root = createWidgetTree(uiaRoot); root.set(Tags.Role, Roles.Process); root.set(Tags.NotResponding, false); - + root.set(Tags.HWND, uiaRoot.children.get(0).get(Tags.HWND)); for (Widget w : root) w.set(Tags.Path,Util.indexString(w)); if (system != null && (root == null || root.childCount() == 0) && system.getNativeAutomationCache() != null) From 8b1c88a9608d8d64240224358bf516124f8623f6 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 28 Apr 2021 15:30:09 +0200 Subject: [PATCH 19/40] Take only visible web elements into account when extracting the expected text. --- .../VisualValidationFactory.java | 3 +- .../VisualValidationSettings.java | 2 + .../visualvalidation/VisualValidator.java | 6 ++- ...or.java => ExpectedTextExtractorBase.java} | 41 +++++++++++-------- .../ExpectedTextExtractorDesktop.java | 12 ++++++ .../ExpectedTextExtractorWebdriver.java | 19 +++++++++ .../extractor/ExtractorFactory.java | 12 +++--- .../src/org/fruit/monkey/DefaultProtocol.java | 4 +- 8 files changed, 74 insertions(+), 25 deletions(-) rename testar/src/nl/ou/testar/visualvalidation/extractor/{ExpectedTextExtractor.java => ExpectedTextExtractorBase.java} (86%) create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorDesktop.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java index c15d95283..e031a762e 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationFactory.java @@ -4,9 +4,10 @@ public class VisualValidationFactory { - public static VisualValidationManager createVisualValidator() { + public static VisualValidationManager createVisualValidator(String protocol) { VisualValidationManager visualValidator; VisualValidationSettings visualValidation = ExtendedSettingsFactory.createVisualValidationSettings(); + visualValidation.protocol = protocol; if (visualValidation.enabled) { visualValidator = new VisualValidator(visualValidation); diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java index 0d4a52724..c58c86167 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java @@ -42,6 +42,8 @@ public class VisualValidationSettings extends ExtendedSettingBase { public Boolean enabled; public OcrConfiguration ocrConfiguration; + // The selected protocol will be set automatically when we initialize TESTAR. + public String protocol; @Override public int compareTo(VisualValidationSettings other) { diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 27b947b14..9abe564a4 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -57,7 +57,11 @@ public VisualValidator(@NonNull VisualValidationSettings settings) { _ocrEngine = null; } - _extractor = ExtractorFactory.CreateTextExtractor(); + if (settings.protocol.contains("webdriver_generic")) { + _extractor = ExtractorFactory.CreateExpectedTextExtractorWebdriver(); + } else { + _extractor = ExtractorFactory.CreateExpectedTextExtractorDesktop(); + } _matcher = VisualMatcherFactory.createLocationMatcher(); } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java similarity index 86% rename from testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java rename to testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java index 965f62fbe..6df68a6d9 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -25,9 +25,8 @@ import java.util.stream.Stream; import static org.fruit.alayer.Tags.*; -import static org.fruit.alayer.webdriver.enums.WdTags.WebTextContent; -public class ExpectedTextExtractor extends Thread implements TextExtractorInterface { +public class ExpectedTextExtractorBase extends Thread implements TextExtractorInterface { private static final String TAG = "ExpectedTextExtractor"; private final String rootElementPath = "[0]"; @@ -37,6 +36,10 @@ public class ExpectedTextExtractor extends Thread implements TextExtractorInterf private org.fruit.alayer.State _state = null; private ExpectedTextCallback _callback = null; + final private Tag defaultTag; + + private boolean _loggingEnabled; + /** * A blacklist for {@link Widget}'s which should be ignored based on their {@code Role} because the don't contain * readable text. Optional when the value (which represents the ancestor path) is set, the {@link Widget} should @@ -58,7 +61,7 @@ public class ExpectedTextExtractor extends Thread implements TextExtractorInterf .filter(tag -> tag.type().equals(String.class)) .collect(Collectors.toMap(Tag::name, tag -> (Tag) tag)); - ExpectedTextExtractor() { + ExpectedTextExtractorBase(Tag defaultTag) { WidgetTextConfiguration config = ExtendedSettingsFactory.createWidgetTextConfiguration(); // Load the extractor configuration into a lookup table for quick access. config.widget.forEach(it -> { @@ -72,6 +75,10 @@ public class ExpectedTextExtractor extends Thread implements TextExtractorInterf } }); + _loggingEnabled = config.loggingEnabled; + + this.defaultTag = defaultTag; + setName(TAG); start(); } @@ -81,7 +88,9 @@ public void ExtractExpectedText(org.fruit.alayer.State state, ExpectedTextCallba synchronized (_threadSync) { _state = state; _callback = callback; - Logger.log(Level.TRACE, TAG, "Queue new text extract."); + if (_loggingEnabled) { + Logger.log(Level.TRACE, TAG, "Queue new text extract."); + } _threadSync.notifyAll(); } } @@ -100,7 +109,9 @@ public void run() { } catch (InterruptedException e) { // Happens if someone interrupts your thread. - Logger.log(Level.INFO, TAG, "Wait interrupted"); + if (_loggingEnabled) { + Logger.log(Level.INFO, TAG, "Wait interrupted"); + } e.printStackTrace(); } } @@ -138,10 +149,9 @@ private void extractText() { List expectedElements = new ArrayList<>(); for (Widget widget : _state) { String widgetRole = widget.get(Role).name(); + String text = widget.get(getVisualTextTag(widgetRole), ""); if (widgetIsIncluded(widget, widgetRole)) { - String text = widget.get(getVisualTextTag(widgetRole), ""); - if (text != null && !text.isEmpty()) { Rectangle absoluteLocation = getLocation(widget); Rectangle relativeLocation = new Rectangle( @@ -152,14 +162,16 @@ private void extractText() { expectedElements.add(new ExpectedElement(relativeLocation, text)); } } else { - Logger.log(Level.DEBUG, TAG, "Widget {} ignored", widgetRole); + if (_loggingEnabled) { + Logger.log(Level.DEBUG, TAG, "Widget {} with role {} is ignored", text, widgetRole); + } } } _callback.ReportExtractedText(expectedElements); } - private boolean widgetIsIncluded(Widget w, String role) { + protected boolean widgetIsIncluded(Widget widget, String role) { boolean containsReadableText = true; if (_blacklist.containsKey(role)) { containsReadableText = false; @@ -167,7 +179,7 @@ private boolean widgetIsIncluded(Widget w, String role) { List blacklistedAncestors = _blacklist.get(role); if (!blacklistedAncestors.isEmpty()) { StringBuilder sb = new StringBuilder(); - Util.ancestors(w).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); + Util.ancestors(widget).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); // Check if we should ignore this widget based on its ancestors. String ancestors = sb.toString(); @@ -181,18 +193,15 @@ private boolean widgetIsIncluded(Widget w, String role) { } private Tag getVisualTextTag(String widgetRole) { - // Check if we have specified a custom tag for this widget, if so convert the string identifier into the actual - // Tag. - // TODO TM: Should inject this based on the selected protocol? - Tag defaultTag = widgetRole.contains("Wd") ? WebTextContent : Title; return _tag.getOrDefault(_lookupTable.getOrDefault(widgetRole, ""), defaultTag); } @Override public void Destroy() { stopAndJoinThread(); - - Logger.log(Level.DEBUG, TAG, "Extractor destroyed."); + if (_loggingEnabled) { + Logger.log(Level.DEBUG, TAG, "Extractor destroyed."); + } } private void stopAndJoinThread() { diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorDesktop.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorDesktop.java new file mode 100644 index 000000000..870a8c978 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorDesktop.java @@ -0,0 +1,12 @@ +package nl.ou.testar.visualvalidation.extractor; + +import org.fruit.alayer.Widget; + +import static org.fruit.alayer.Tags.Title; + +public class ExpectedTextExtractorDesktop extends ExpectedTextExtractorBase { + + ExpectedTextExtractorDesktop() { + super(Title); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java new file mode 100644 index 000000000..95f330f19 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java @@ -0,0 +1,19 @@ +package nl.ou.testar.visualvalidation.extractor; + +import org.fruit.alayer.Widget; + +import static org.fruit.alayer.webdriver.enums.WdTags.WebIsFullOnScreen; +import static org.fruit.alayer.webdriver.enums.WdTags.WebTextContent; + +class ExpectedTextExtractorWebdriver extends ExpectedTextExtractorBase { + + ExpectedTextExtractorWebdriver() { + super(WebTextContent); + } + + @Override + protected boolean widgetIsIncluded(Widget widget, String role) { + // Check if the widget is visible + return widget.get(WebIsFullOnScreen) && super.widgetIsIncluded(widget, role); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java index 3ae7aa559..9584eb32f 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExtractorFactory.java @@ -1,15 +1,15 @@ package nl.ou.testar.visualvalidation.extractor; public class ExtractorFactory { - public static TextExtractorInterface CreateTextExtractor(){ - return CreateExpectedTextExtractor(); + public static TextExtractorInterface CreateDummyExtractor(){ + return new DummyExtractor(); } - private static TextExtractorInterface CreateDummyExtractor(){ - return new DummyExtractor(); + public static TextExtractorInterface CreateExpectedTextExtractorDesktop(){ + return new ExpectedTextExtractorDesktop(); } - private static TextExtractorInterface CreateExpectedTextExtractor(){ - return new ExpectedTextExtractor(); + public static TextExtractorInterface CreateExpectedTextExtractorWebdriver(){ + return new ExpectedTextExtractorWebdriver(); } } diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index c8e2517c4..5f5ad5626 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -40,6 +40,7 @@ import static org.fruit.alayer.Tags.OracleVerdict; import static org.fruit.alayer.Tags.SystemState; import static org.fruit.monkey.ConfigTags.LogLevel; +import static org.fruit.monkey.ConfigTags.ProtocolClass; import java.awt.*; import java.io.BufferedInputStream; @@ -102,6 +103,7 @@ import org.jnativehook.GlobalScreen; import org.jnativehook.NativeHookException; import org.openqa.selenium.SessionNotCreatedException; +import org.testar.Logger; import org.testar.OutputStructure; public class DefaultProtocol extends RuntimeControlsProtocol { @@ -364,7 +366,7 @@ protected void initialize(Settings settings) { // new state model manager stateModelManager = StateModelManagerFactory.getStateModelManager(settings); - visualValidationManager = VisualValidationFactory.createVisualValidator(); + visualValidationManager = VisualValidationFactory.createVisualValidator(settings.get(ProtocolClass)); } try { From 3379a46e34f7884e60d59ed9a6041e08ff567c0b Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Thu, 10 Jun 2021 21:32:29 +0200 Subject: [PATCH 20/40] Fixed parsing screenshots from webdriver protocol. --- testar/build.gradle | 1 + .../extractor/ExpectedTextExtractorBase.java | 9 +- .../extractor/WidgetTextConfiguration.java | 11 ++- .../ocr/tesseract/TesseractOcrEngine.java | 88 ++++++++++++++++--- .../ocr/tesseract/TesseractResult.java | 6 ++ .../ocr/tesseract/TesseractSettings.java | 12 ++- .../fruit/alayer/webdriver/WdScreenshot.java | 25 +++--- 7 files changed, 118 insertions(+), 34 deletions(-) diff --git a/testar/build.gradle b/testar/build.gradle index 7bb93d9d7..95444c357 100644 --- a/testar/build.gradle +++ b/testar/build.gradle @@ -89,6 +89,7 @@ dependencies { compile group: 'org.eclipse.jetty', name: 'apache-jsp', version: '9.4.30.v20200611' compile group: 'org.eclipse.jetty', name: 'apache-jstl', version: '9.4.30.v20200611' compile group: 'org.bytedeco', name: 'tesseract-platform', version: '4.1.1-1.5.4' + compile group: 'org.bytedeco', name: 'javacv', version: '1.5.5' compile group: 'commons-io', name: 'commons-io', version: '2.7' runtime project(':windows') } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java index 6df68a6d9..883c781c6 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -28,7 +28,6 @@ public class ExpectedTextExtractorBase extends Thread implements TextExtractorInterface { private static final String TAG = "ExpectedTextExtractor"; - private final String rootElementPath = "[0]"; AtomicBoolean running = new AtomicBoolean(true); @@ -38,7 +37,7 @@ public class ExpectedTextExtractorBase extends Thread implements TextExtractorIn final private Tag defaultTag; - private boolean _loggingEnabled; + private final boolean _loggingEnabled; /** * A blacklist for {@link Widget}'s which should be ignored based on their {@code Role} because the don't contain @@ -108,10 +107,7 @@ public void run() { extractText(); } catch (InterruptedException e) { - // Happens if someone interrupts your thread. - if (_loggingEnabled) { - Logger.log(Level.INFO, TAG, "Wait interrupted"); - } + Logger.log(Level.ERROR, TAG, "Wait interrupted"); e.printStackTrace(); } } @@ -138,6 +134,7 @@ private void extractText() { // Acquire the absolute location of the SUT on the screen. Rectangle applicationPosition = null; for (Widget widget : _state) { + String rootElementPath = "[0]"; if (widget.get(Path).contentEquals(rootElementPath)) { applicationPosition = getLocation(widget); break; diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java index 529cc1b8c..b7a91f2ac 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/WidgetTextConfiguration.java @@ -14,10 +14,12 @@ @XmlAccessorType(XmlAccessType.FIELD) public class WidgetTextConfiguration extends ExtendedSettingBase { List widget; + boolean loggingEnabled; public static WidgetTextConfiguration CreateDefault() { WidgetTextConfiguration instance = new WidgetTextConfiguration(); + // Desktop protocol WidgetTextSetting scrollBar = WidgetTextSetting.CreateIgnore("UIAScrollBar"); WidgetTextSetting menuBar = WidgetTextSetting.CreateIgnore("UIAMenuBar"); WidgetTextSetting statusBar = WidgetTextSetting.CreateIgnore("UIAStatusBar"); @@ -29,7 +31,12 @@ public static WidgetTextConfiguration CreateDefault() { WidgetTextSetting scrollBarButtons = WidgetTextSetting.CreateIgnoreAncestorBased("UIAButton", Arrays.asList("UIAScrollBar", "UIAEdit", "UIAWindow", "Process")); - instance.widget = new ArrayList<>(Arrays.asList(scrollBar, statusBar, menuBar, textEdit, icon, toolBarButtons, scrollBarButtons)); + // Webdriver protocol + WidgetTextSetting skipToContentWdou = WidgetTextSetting.CreateIgnoreAncestorBased("WdA", + Arrays.asList("WdDIV", "Process")); + + instance.widget = new ArrayList<>(Arrays.asList(scrollBar, statusBar, menuBar, textEdit, icon, toolBarButtons, scrollBarButtons, skipToContentWdou)); + instance.loggingEnabled = false; return instance; } @@ -37,7 +44,7 @@ public static WidgetTextConfiguration CreateDefault() { public int compareTo(WidgetTextConfiguration other) { int result = -1; - if (widget.size() == other.widget.size()) { + if (widget.size() == other.widget.size() && loggingEnabled == other.loggingEnabled) { for (int i = 0; i < widget.size(); i++) { int index = i; if (other.widget.stream().noneMatch(it -> it.compareTo(widget.get(index)) == 0)) { diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java index 3af3520f7..6b300bf9a 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java @@ -4,6 +4,8 @@ import nl.ou.testar.visualvalidation.ocr.OcrResultCallback; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.apache.logging.log4j.Level; +import org.bytedeco.javacv.Java2DFrameConverter; +import org.bytedeco.javacv.LeptonicaFrameConverter; import org.bytedeco.tesseract.ETEXT_DESC; import org.bytedeco.tesseract.ResultIterator; import org.bytedeco.tesseract.TessBaseAPI; @@ -12,19 +14,27 @@ import org.testar.Logger; import org.testar.settings.ExtendedSettingsFactory; +import javax.imageio.ImageIO; import java.awt.image.BufferedImage; import java.awt.image.DataBuffer; import java.awt.image.DataBufferByte; import java.awt.image.DataBufferInt; import java.awt.image.DataBufferShort; import java.awt.image.DataBufferUShort; +import java.io.File; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.IntBuffer; import java.nio.ShortBuffer; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static java.awt.image.BufferedImage.TYPE_4BYTE_ABGR; +import static java.awt.image.BufferedImage.TYPE_INT_ARGB; +import static org.bytedeco.leptonica.global.lept.pixRead; +import static org.bytedeco.tesseract.global.tesseract.PSM_AUTO_OSD; /** * OCR engine implementation dedicated for the Tesseract engine. @@ -37,12 +47,12 @@ */ public class TesseractOcrEngine extends Thread implements OcrEngineInterface { private static final String TAG = "Tesseract"; - - AtomicBoolean running = new AtomicBoolean(true); - private final TessBaseAPI _engine; private final Boolean _scanSync = true; private final int _imageResolution; + private final boolean _loggingEnabled; + private final boolean _saveImageBufferToDisk; + AtomicBoolean running = new AtomicBoolean(true); private BufferedImage _image = null; private OcrResultCallback _callback = null; @@ -50,13 +60,17 @@ public TesseractOcrEngine() { _engine = new TessBaseAPI(); TesseractSettings config = ExtendedSettingsFactory.createTesseractSetting(); _imageResolution = config.imageResolution; + _loggingEnabled = config.loggingEnabled; + _saveImageBufferToDisk = config.saveImageBufferToDisk; if (_engine.Init(config.dataPath, config.language) != 0) { Logger.log(Level.ERROR, TAG, "Could not initialize tesseract."); } - Logger.log(Level.INFO, TAG, "Tesseract engine created; Language:{} Data path:{}", - config.language, config.dataPath); + if (_loggingEnabled) { + Logger.log(Level.INFO, TAG, "Tesseract engine created; Language:{} Data path:{}", + config.language, config.dataPath); + } setName(TAG); start(); @@ -67,7 +81,9 @@ public void AnalyzeImage(BufferedImage image, OcrResultCallback callback) { synchronized (_scanSync) { _image = image; _callback = callback; - Logger.log(Level.TRACE, TAG, "Queue new image scan."); + if (_loggingEnabled) { + Logger.log(Level.TRACE, TAG, "Queue new image scan."); + } _scanSync.notifyAll(); } } @@ -85,8 +101,7 @@ public void run() { recognizeText(); } catch (InterruptedException e) { - // Happens if someone interrupts your thread. - Logger.log(Level.INFO, TAG, "Wait interrupted"); + Logger.log(Level.ERROR, TAG, "Wait interrupted"); e.printStackTrace(); } } @@ -98,8 +113,11 @@ private void recognizeText() { Logger.log(Level.ERROR, TAG, "Should not try to detect text on empty image/callback"); return; } + List recognizedWords = new ArrayList<>(); + _engine.SetPageSegMode(PSM_AUTO_OSD); + loadImageIntoEngine(_image); if (_engine.Recognize(new ETEXT_DESC()) != 0) { @@ -113,11 +131,32 @@ private void recognizeText() { } } _engine.Clear(); - // Notify the callback with the discovered words. - _callback.reportResult(recognizedWords); + + // Filter out the empty items and notify the callback with the discovered words. + _callback.reportResult(recognizedWords.stream() + .filter(recognizedElement -> !recognizedElement._text.isEmpty()) + .collect(Collectors.toList()) + ); } private void loadImageIntoEngine(@NonNull BufferedImage image) { + // Ideally we use the raw data but unfortunately the webdriver screenshots have a different format which doesn't + // work with the current conversion for loading a raw buffer. + switch (image.getType()) { + case TYPE_INT_ARGB: + // Works for desktop protocol. + loadImageAsRawData(image); + break; + case TYPE_4BYTE_ABGR: + // Works for webdriver protocol. + loadImageAsPix(image); + break; + default: + throw new IllegalArgumentException("Loading OCR image not supported for image type:" + image.getType()); + } + } + + private void loadImageAsRawData(@NonNull BufferedImage image) { DataBuffer dataBuffer = image.getData().getDataBuffer(); ByteBuffer byteBuffer; @@ -148,12 +187,37 @@ private void loadImageIntoEngine(@NonNull BufferedImage image) { _engine.SetSourceResolution(_imageResolution); } + private void loadImageAsPix(@NonNull BufferedImage image) { + // Works for web driver + if (_saveImageBufferToDisk) { + try { + File outputFile = File.createTempFile("testar", ".png"); + outputFile.deleteOnExit(); + ImageIO.write(_image, "png", outputFile); + _engine.SetImage(pixRead(outputFile.getAbsolutePath())); + _engine.SetSourceResolution(_imageResolution); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + try (Java2DFrameConverter converter = new Java2DFrameConverter(); + LeptonicaFrameConverter converter2 = new LeptonicaFrameConverter()) { + // Convert the buffered image into a PIX. + _engine.SetImage(converter2.convert(converter.convert(image))); + } catch (Exception e) { + Logger.log(Level.ERROR, TAG, "Failed to convert buffered image into PIX"); + } + } + } + @Override public void Destroy() { stopAndJoinThread(); _engine.End(); - Logger.log(Level.DEBUG, TAG, "Engine destroyed."); + if (_loggingEnabled) { + Logger.log(Level.DEBUG, TAG, "Engine destroyed."); + } } private void stopAndJoinThread() { diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java index 57b3f5b7c..ced39ce7a 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java @@ -1,9 +1,11 @@ package nl.ou.testar.visualvalidation.ocr.tesseract; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; +import org.apache.logging.log4j.Level; import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.IntPointer; import org.bytedeco.tesseract.ResultIterator; +import org.testar.Logger; import java.awt.Rectangle; import java.util.function.Supplier; @@ -25,6 +27,10 @@ static RecognizedElement Extract(ResultIterator recognizedElement, int granulari Supplier intPointerSupplier = () -> new IntPointer(new int[1]); BytePointer ocrResult = recognizedElement.GetUTF8Text(granularity); + if (ocrResult == null){ + Logger.log(Level.ERROR, "OCR", "Results is null"); + return new RecognizedElement(new Rectangle(), 0, ""); + } String recognizedText = ocrResult.getString().trim(); float confidence = recognizedElement.Confidence(granularity); diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java index ead0f49c6..be1e9715e 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractSettings.java @@ -12,19 +12,27 @@ public class TesseractSettings extends ExtendedSettingBase { public String dataPath; public String language; public int imageResolution; + public boolean loggingEnabled; + public boolean saveImageBufferToDisk; public static TesseractSettings CreateDefault() { TesseractSettings instance = new TesseractSettings(); instance.dataPath = System.getenv("LOCALAPPDATA") + "\\Tesseract-OCR\\tessdata"; instance.language = "eng"; - instance.imageResolution = 160; + instance.imageResolution = 144; + instance.loggingEnabled = false; + instance.saveImageBufferToDisk = true; return instance; } @Override public int compareTo(TesseractSettings other) { int result = -1; - if ((language.contentEquals(other.language) && (dataPath.contentEquals(other.dataPath)))) { + if (language.contentEquals(other.language) + && (dataPath.contentEquals(other.dataPath)) + && (imageResolution == other.imageResolution) + && (loggingEnabled == other.loggingEnabled) + && (saveImageBufferToDisk == other.saveImageBufferToDisk)) { result = 0; } return result; diff --git a/webdriver/src/org/fruit/alayer/webdriver/WdScreenshot.java b/webdriver/src/org/fruit/alayer/webdriver/WdScreenshot.java index 16e66a969..adddae725 100644 --- a/webdriver/src/org/fruit/alayer/webdriver/WdScreenshot.java +++ b/webdriver/src/org/fruit/alayer/webdriver/WdScreenshot.java @@ -39,7 +39,7 @@ import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.io.File; +import java.io.ByteArrayInputStream; /** * Extend AWTCanvas to get the screenshot from WebDriver @@ -52,21 +52,22 @@ private WdScreenshot() { } public static WdScreenshot fromScreenshot(Rect r, long windowHandle) - throws StateBuildException { + throws StateBuildException { WdScreenshot wdScreenshot = new WdScreenshot(); RemoteWebDriver webDriver = WdDriver.getRemoteWebDriver(); try { - File screenshot = webDriver.getScreenshotAs(OutputType.FILE); - BufferedImage fullImg = ImageIO.read(screenshot); - double displayScale = Environment.getInstance().getDisplayScale(windowHandle); - int x = (int) Math.max(0, r.x() * displayScale); - int y = (int) Math.max(0, r.y() * displayScale); - int width = (int) Math.min(fullImg.getWidth(), r.width() * displayScale); - int height = (int) Math.min(fullImg.getHeight(), r.height() * displayScale); - wdScreenshot.img = fullImg.getSubimage(x, y, width, height); - } - catch (Exception ignored) { + byte[] image = webDriver.getScreenshotAs(OutputType.BYTES); + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(image)) { + BufferedImage fullImg = ImageIO.read(inputStream); + double displayScale = Environment.getInstance().getDisplayScale(windowHandle); + int x = (int) Math.max(0, r.x() * displayScale); + int y = (int) Math.max(0, r.y() * displayScale); + int width = (int) Math.min(fullImg.getWidth(), r.width() * displayScale); + int height = (int) Math.min(fullImg.getHeight(), r.height() * displayScale); + wdScreenshot.img = fullImg.getSubimage(x, y, width, height); + } + } catch (Exception ignored) { } return wdScreenshot; From 43a76604699da574a8aecf2f4319fea6fb4e17b6 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 12 Jun 2021 21:03:10 +0200 Subject: [PATCH 21/40] Polished matcher algorithm. --- .../matcher/CharacterMatch.java | 26 +++++++++ .../matcher/CharacterMatchEntry.java | 57 +++++++++++++++++++ .../matcher/CharacterMatchResult.java | 11 ++++ .../matcher/ExpectedTextMatchResult.java | 35 ++++++++++++ .../matcher/RecognizedTextMatchResult.java | 37 ++++++++++++ 5 files changed, 166 insertions(+) create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java new file mode 100644 index 000000000..a82278bef --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java @@ -0,0 +1,26 @@ +package nl.ou.testar.visualvalidation.matcher; + +/** + * Stores the outcome of the matcher result for the given character. + */ +public class CharacterMatch { + CharacterMatchResult result; + final CharacterMatchEntry character; + + /** + * Constructor. + * @param character The character which we try to match. + */ + public CharacterMatch(char character){ + result = CharacterMatchResult.NO_MATCH; + this.character = new CharacterMatchEntry(character); + } + + @Override + public String toString() { + return "CharacterMatch{" + + "result=" + result + + ", ch=" + character + + '}'; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java new file mode 100644 index 000000000..b6cb74a68 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java @@ -0,0 +1,57 @@ +package nl.ou.testar.visualvalidation.matcher; + +import org.checkerframework.checker.nullness.qual.NonNull; + +/** + * Container to store the matched counter part of the expected character. + */ +public class CharacterMatchEntry { + final Character character; + CharacterMatchEntry match; + + /** + * Constructor. + * + * @param character The character which we try to match. + */ + CharacterMatchEntry(char character) { + this.character = character; + match = null; + } + + /** + * Mark this entry as matched. + * + * @param matchedCharacter The matched character. + */ + public void Match(@NonNull CharacterMatchEntry matchedCharacter) { + match = matchedCharacter; + matchedCharacter.match = this; + } + + /** + * Check if the character has not matched. + * + * @return True when not matched. + */ + public boolean isNotMatched() { + return match == null; + } + + /** + * Check if the character is matched. + * + * @return True when matched. + */ + public boolean isMatched() { + return !isNotMatched(); + } + + @Override + public String toString() { + return "CharacterMatchEntry{" + + "match=" + Integer.toHexString(System.identityHashCode(match)) + + ", character=" + character + + '}'; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java new file mode 100644 index 000000000..ebe28282e --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java @@ -0,0 +1,11 @@ +package nl.ou.testar.visualvalidation.matcher; + +/** + * The match result of an individual character. + */ +enum CharacterMatchResult { + MATCHED, + CASE_MISMATCH, + WHITESPACE_CORRECTED, + NO_MATCH +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java new file mode 100644 index 000000000..f02c6e022 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java @@ -0,0 +1,35 @@ +package nl.ou.testar.visualvalidation.matcher; + +import java.util.ArrayList; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * The matcher result for the expected text. + */ +public class ExpectedTextMatchResult { + ArrayList expectedText; + + /** + * Constructor. + * @param expectedText The expected text that we are matchin. + */ + public ExpectedTextMatchResult(String expectedText) { + this.expectedText = new ArrayList<>(expectedText.length()); + for (char ch: expectedText.toCharArray()) { + this.expectedText.add(new CharacterMatch(ch)); + } + } + + @Override + public String toString() { + return "ExpectedTextMatchResult (" + + expectedText.stream().map(e -> e.character.character).collect(Collector.of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString)) + + ":" + expectedText.size() + "){ " + + expectedText.stream().map(CharacterMatch::toString).collect(Collectors.joining()); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java new file mode 100644 index 000000000..4b6650c3b --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java @@ -0,0 +1,37 @@ +package nl.ou.testar.visualvalidation.matcher; + +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * The matcher result for the recognized text. + */ +public class RecognizedTextMatchResult { + List recognized; + + /** + * Constructor creates a consecutive order of recognized text characters. + * @param recognizedElements A list with recognized elements that should be treated as one. + */ + public RecognizedTextMatchResult(List recognizedElements) { + recognized = recognizedElements.stream().flatMap( + recognizedElement -> recognizedElement._text.chars().boxed().map(it -> Character.toChars(it)[0])) + .map(CharacterMatchEntry::new) + .collect(Collectors.toList()); + } + + @Override + public String toString() { + return "RecognizedTextMatchResult{" + + recognized.stream().map(e -> e.character).collect(Collector.of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString)) + + ":" + recognized.size() + "){ " + + recognized.stream().map(CharacterMatchEntry::toString).collect(Collectors.joining()); + } +} From afd73e7739cb20ff420a00d1437abd6435fb3c6d Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 23 Jun 2021 16:40:52 +0200 Subject: [PATCH 22/40] Improved matcher result --- .../VisualValidationSettings.java | 23 +- .../visualvalidation/VisualValidator.java | 7 +- .../matcher/ContentMatchResult.java | 83 +++++++ .../matcher/ContentMatcher.java | 143 +++++++++++ .../matcher/ExpectedTextMatchResult.java | 2 +- .../{Match.java => LocationMatch.java} | 6 +- .../matcher/LocationMatcher.java | 232 ++++++------------ .../matcher/MatcherConfiguration.java | 33 +++ .../matcher/MatcherResult.java | 28 ++- .../matcher/VisualMatcherFactory.java | 4 +- 10 files changed, 382 insertions(+), 179 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java rename testar/src/nl/ou/testar/visualvalidation/matcher/{Match.java => LocationMatch.java} (86%) create mode 100644 testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java index c58c86167..39458e6f2 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationSettings.java @@ -30,6 +30,7 @@ package nl.ou.testar.visualvalidation; +import nl.ou.testar.visualvalidation.matcher.MatcherConfiguration; import nl.ou.testar.visualvalidation.ocr.OcrConfiguration; import org.testar.settings.ExtendedSettingBase; @@ -42,14 +43,25 @@ public class VisualValidationSettings extends ExtendedSettingBase { public Boolean enabled; public OcrConfiguration ocrConfiguration; + public MatcherConfiguration matcherConfiguration; // The selected protocol will be set automatically when we initialize TESTAR. public String protocol; + public static VisualValidationSettings CreateDefault() { + VisualValidationSettings defaultInstance = new VisualValidationSettings(); + defaultInstance.enabled = true; + defaultInstance.ocrConfiguration = OcrConfiguration.CreateDefault(); + defaultInstance.matcherConfiguration = MatcherConfiguration.CreateDefault(); + return defaultInstance; + } + @Override public int compareTo(VisualValidationSettings other) { int res = -1; if ((enabled.equals(other.enabled)) && - (ocrConfiguration.compareTo(other.ocrConfiguration) == 0)) { + (ocrConfiguration.compareTo(other.ocrConfiguration) == 0) && + (matcherConfiguration.compareTo(other.matcherConfiguration) == 0) + ) { res = 0; } return res; @@ -59,13 +71,8 @@ public int compareTo(VisualValidationSettings other) { public String toString() { return "VisualValidationSettings{" + "enabled=" + enabled + + ", ocr=" + ocrConfiguration + + ", matcher=" + matcherConfiguration + '}'; } - - public static VisualValidationSettings CreateDefault() { - VisualValidationSettings DefaultInstance = new VisualValidationSettings(); - DefaultInstance.enabled = true; - DefaultInstance.ocrConfiguration = OcrConfiguration.CreateDefault(); - return DefaultInstance; - } } diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 9abe564a4..e24113b9d 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -63,7 +63,7 @@ public VisualValidator(@NonNull VisualValidationSettings settings) { _extractor = ExtractorFactory.CreateExpectedTextExtractorDesktop(); } - _matcher = VisualMatcherFactory.createLocationMatcher(); + _matcher = VisualMatcherFactory.createLocationMatcher(settings.matcherConfiguration); } @Override @@ -138,13 +138,16 @@ private void updateVerdict(State state) { if (_matcherResult != null) { // Analysis the raw result and create a verdict. - _matcherResult.getMatches(); + _matcherResult.getResult().forEach(result -> + Logger.log(Level.INFO, TAG, "Content match result: {}", result) + ); Verdict result = new Verdict(Verdict.SEVERITY_WARNING, "Not all texts has been recognized", new TextVisualizer(new AbsolutePosition(10, 10), "->", RedPen)); } else { // Set verdict to failure we should have a matcher result as minimal input. + Logger.log(Level.INFO, TAG, "No result"); } Logger.log(Level.INFO, TAG, "Updating verdict {}"); } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java new file mode 100644 index 000000000..8073c7083 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java @@ -0,0 +1,83 @@ +package nl.ou.testar.visualvalidation.matcher; + +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * Holds the matching result for the content of the expected text. + */ +public class ContentMatchResult { + public final ExpectedTextMatchResult expectedResult; + public final RecognizedTextMatchResult recognizedResult; + public final long totalMatched; + public final long totalExpected; + + public ContentMatchResult(ExpectedTextMatchResult expectedResult, RecognizedTextMatchResult recognizedResult) { + this.expectedResult = expectedResult; + this.recognizedResult = recognizedResult; + totalMatched = expectedResult.expectedText.stream() + .filter(e -> e.result != CharacterMatchResult.NO_MATCH) + .count(); + totalExpected = expectedResult.expectedText.size(); + } + + @Override + public String toString() { + StringBuilder str = new StringBuilder() + .append("\n") + + .append("Matched \"").append(expectedResult.expectedText.stream(). + map(e -> e.character.character).collect(Collector.of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString))) + .append("\"\n") + + .append("Result: ") + .append(expectedResult.expectedText.stream().map(it -> { + switch (it.result) { + case WHITESPACE_CORRECTED: + return "W"; + case CASE_MISMATCH: + return "C"; + case MATCHED: + return "V"; + default: + return "X"; + } + }).collect(Collectors.toList())) + .append(" [").append(totalMatched).append("/").append(totalExpected).append("]\n") + + .append("Expect: ").append(expectedResult.expectedText.stream() + .map(it -> it.character.character) + .collect(Collectors.toList())) + .append("\n") + + .append("Found: ") + .append(expectedResult.expectedText.stream() + .map(it -> { + if (it.character.isMatched()) { + return it.character.match.character; + } else { + return " "; + } + }) + .collect(Collectors.toList())) + .append("\n"); + + List garbage = recognizedResult.recognized.stream() + .filter(CharacterMatchEntry::isNotMatched) + .collect(Collectors.toList()); + + if (!garbage.isEmpty()) { + str.append("Garbage:"); + str.append(garbage.stream() + .map(it -> it.character) + .collect(Collectors.toList())); + } + + return str.toString(); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java new file mode 100644 index 000000000..a52183f57 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java @@ -0,0 +1,143 @@ +package nl.ou.testar.visualvalidation.matcher; + +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; +import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.fruit.Pair; +import org.testar.Logger; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class ContentMatcher { + static final String TAG = "ContentMatcher"; + + /** + * Tries to match characters from the recognized text with the expected text characters. Recognized text elements + * are first sorted based on their location. By calculating the line height of text the recognized text elements are + * first grouped per text line. After that they are sorted based on their x coordinate. The result of this sorting + * mechanism is a grid from left to right and top to bottom. The algorithm tries to find a matching character from + * the grid, once found the index is stored and will be used as the starting position for the next iteration. + */ + public static ContentMatchResult Match(LocationMatch locationMatch) { + String expectedText = locationMatch.expectedElement._text; + ExpectedTextMatchResult expectedResult = new ExpectedTextMatchResult(expectedText); + Logger.log(Level.INFO, TAG, "Expected text @{} : {}", locationMatch.expectedElement._location, expectedResult); + + // Sort the recognized elements. + List sorted = sortRecognizedElements(locationMatch); + RecognizedTextMatchResult recognizedResult = new RecognizedTextMatchResult(sorted); + Logger.log(Level.INFO, TAG, "Recognized text: {}", recognizedResult); + + // Iterate over the expected text and try to match with a recognized element character. + int unMatchedSize = recognizedResult.recognized.size(); + int indexCounter = 0; + for (CharacterMatch expectedChar : expectedResult.expectedText) { + char actual = expectedChar.character.character; + + // Iterate over the recognized characters. + for (int k = indexCounter; k < unMatchedSize; k++) { + CharacterMatchEntry item = recognizedResult.recognized.get(k); + + // Try to match the actual char case sensitive: + if (item.character == actual && item.isNotMatched()) { + expectedChar.character.Match(item); + expectedChar.result = CharacterMatchResult.MATCHED; + // We have found a match move the index counter. + indexCounter = k + 1; + break; + } + + // Try to match the actual char as case insensitive: + if (Character.isLetter(actual)) { + int CASING = 32; + if (Math.abs(item.character.compareTo(actual)) == CASING && item.isNotMatched()) { + expectedChar.character.Match(item); + expectedChar.result = CharacterMatchResult.CASE_MISMATCH; + // We have found a match inside the unMatched List move one position. + indexCounter = k + 1; + break; + } + } + } + } + + correctWhitespaces(expectedResult); + + return new ContentMatchResult(expectedResult, recognizedResult); + } + + @NonNull + private static List sortRecognizedElements(LocationMatch locationMatch) { + Map> lineBucket = sortRecognizedElementsPerTextLine(locationMatch); + + // Get the sorted Y-axe coordinates for the identified lines. + List bucketSorted = lineBucket.keySet().stream().sorted().collect(Collectors.toList()); + + // For each line sort the elements based on their X-axis coordinate and add the to the result. + List sorted = new ArrayList<>(); + bucketSorted.forEach(line -> + { + List sort = lineBucket.get(line).stream() + .sorted(Comparator.comparingInt(o -> o._location.x)) + .collect(Collectors.toList()); + sorted.addAll(sort); + } + ); + return sorted; + } + + @NonNull + private static Map> sortRecognizedElementsPerTextLine(LocationMatch locationMatch) { + // Determine the average height of the recognized text elements. + double lineHeight = locationMatch.recognizedElements.stream() + .mapToInt(e -> e._location.height) + .average() + .orElse(Double.NaN); + + // Create a overview of pairs reflecting the center line of the text and the recognized element. + List> lines = locationMatch.recognizedElements.stream() + .map(e -> new Pair<>(e._location.y + (e._location.height / 2), e)) + .collect(Collectors.toList()); + + final int margin = (int) Math.round(lineHeight / 2); + Map> lineBucket = new HashMap<>(); + + // Find a matching line within the margins of the center line, if not found create a new line. + for (Pair line : lines) { + boolean foundBucket = false; + for (Integer c : lineBucket.keySet()) { + if (IntStream.rangeClosed(c - margin, c + margin) + .boxed() + .collect(Collectors.toList()) + .contains(line.left())) + { + Logger.log(Level.DEBUG, TAG, "Line {} {} in range of bucket {}", line.left(), line.right(), c); + lineBucket.get(c).add(line.right()); + foundBucket = true; + break; + } + } + if (!foundBucket) { + Logger.log(Level.DEBUG, TAG, "No bucket found creating new one for {} {}", line.left(), line.right()); + lineBucket.put(line.left(), new ArrayList<>()); + lineBucket.get(line.left()).add(line.right()); + } + } + return lineBucket; + } + + private static void correctWhitespaces(ExpectedTextMatchResult expectedResult) { + // Whitespaces are automatically corrected. + for (CharacterMatch res : expectedResult.expectedText) { + if (res.result == CharacterMatchResult.NO_MATCH && Character.isWhitespace(res.character.character)) { + res.result = CharacterMatchResult.WHITESPACE_CORRECTED; + } + } + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java index f02c6e022..bdb1d3b7a 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java @@ -12,7 +12,7 @@ public class ExpectedTextMatchResult { /** * Constructor. - * @param expectedText The expected text that we are matchin. + * @param expectedText The expected text that we are matching. */ public ExpectedTextMatchResult(String expectedText) { this.expectedText = new ArrayList<>(expectedText.length()); diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/Match.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatch.java similarity index 86% rename from testar/src/nl/ou/testar/visualvalidation/matcher/Match.java rename to testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatch.java index 452867586..f8f565646 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/Match.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatch.java @@ -6,13 +6,13 @@ import java.util.HashSet; import java.util.Set; -public class Match { +public class LocationMatch { final public MatchLocation location; final public ExpectedElement expectedElement; public Set recognizedElements = new HashSet<>(); - public Match(ExpectedElement expectedElement, int margin) { + public LocationMatch(ExpectedElement expectedElement, int margin) { this.location = new MatchLocation(margin, expectedElement._location); this.expectedElement = expectedElement; } @@ -23,7 +23,7 @@ public void addRecognizedElement(RecognizedElement element) { @Override public String toString() { - return "Match{" + + return "LocationMatch{" + "location=" + location + ", expectedElement=" + expectedElement + ", recognizedElements=" + recognizedElements + diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java index a90d5c555..4c166cc86 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java @@ -3,13 +3,12 @@ import nl.ou.testar.visualvalidation.extractor.ExpectedElement; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.apache.logging.log4j.Level; -import org.fruit.Pair; +import org.checkerframework.checker.nullness.qual.Nullable; import org.testar.Logger; import java.awt.Dimension; import java.awt.Rectangle; import java.util.ArrayList; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -17,181 +16,106 @@ public class LocationMatcher implements VisualMatcher { static final String TAG = "Matcher"; + private final MatcherConfiguration config; + public LocationMatcher(MatcherConfiguration setting) { + config = setting; + } + + @Override + public void destroy() { + + } + + /** + * Match the recognized text elements with the expected text elements based on their location. For each expected + * text element we first link the related recognized text elements based on their location. This process is executed + * based on the surface area of the expected text in the order from small to large. This prevents that we don't + * accidentally map a recognized text element to a different expected element that may overlap the actual expected + * text element. Once we have finalized the linking we start the matching of the actual content. + * + * @param recognizedElements A list of recognized elements. + * @param expectedElements A list of expected text elements. + * @return The result of the matching algorithm. + */ @Override - public MatcherResult Match(List ocrResult, List expectedText) { - MatcherResult result = new MatcherResult(); - List _ocrResult = new ArrayList<>(ocrResult); - - // We first match them based on their location, if they intersect we mark them as match. - // Because there are windows and frames included as well we need to make sure that we don't assign them to - // these elements before we can assign them to the actual widget which is presented on the panel/window. - expectedText.sort((element1, element2) -> { - // Sort based on the area size of the element, from small to big + public MatcherResult Match(List recognizedElements, List expectedElements) { + // 1) Sort based on the area size of the element, from small to big + expectedElements.sort((element1, element2) -> { Dimension element1Size = element1._location.getSize(); Dimension element2Size = element2._location.getSize(); return ((element1Size.width * element1Size.height) - (element2Size.width * element2Size.height)); }); - expectedText.forEach(expectedElement -> { - final Match[] match = {null}; - Set removeRecognizedList = new HashSet<>(); - _ocrResult.forEach(it -> { - // TODO TM: Lookup margin via extended settings framework - int margin = 0; - Rectangle areaOfInterest = it._location; - areaOfInterest.setBounds( - areaOfInterest.x - margin, - areaOfInterest.y - margin, - areaOfInterest.width + (2 * margin), - areaOfInterest.height + (2 * margin) - ); - - // If the recognized element lay inside the area of interest. - if (areaOfInterest.intersects(expectedElement._location)) { - // Prepare to make a match. - if (match[0] == null) { - match[0] = new Match(expectedElement, margin); - } - // TODO TM: If all expected items are found should we break? Or what if we find other items that intersect?! - match[0].addRecognizedElement(it); - } - }); + MatcherResult _matchResult = new MatcherResult(); + List _recognizedElements = new ArrayList<>(recognizedElements); + // 2) Match the OCR results based on their location. + expectedElements.forEach(expectedElement -> { + final LocationMatch locationMatch = getIntersectedElements(_recognizedElements, expectedElement); // All the recognized elements which lay inside the area of interest are linked to this match. - if (match[0] != null) { - result.addMatch(match[0]); + if (locationMatch != null) { + _matchResult.addLocationMatch(locationMatch); - // Now that we have collected all the recognized elements, try to match the content. - analyze(match[0]); + // 3) Now that we have collected all the recognized elements, try to match the content. + _matchResult.addContentMatchResult(ContentMatcher.Match(locationMatch)); - // Remove all the recognized items that have bene linked to this match from the list with items which + // Remove all the recognized items that have been linked to this match from the list with items which // we still need to match based on their location - match[0].recognizedElements.forEach(_ocrResult::remove); + locationMatch.recognizedElements.forEach(_recognizedElements::remove); + } else { + _matchResult.addNoLocationMatch(expectedElement); } }); - Set removeRecognizedList = new HashSet<>(); - result.getMatches().forEach(it -> removeRecognizedList.addAll(it.recognizedElements)); - // Remove the elements which have been matched with expected elements. - removeRecognizedList.forEach(_ocrResult::remove); - - // The remainders in OCR result can be added to no match? - _ocrResult.forEach(result::addNoMatch); - - Logger.log(Level.INFO, TAG, "Result {}", result); - return result; - } - - @Override - public void destroy() { - + // Compose the final matching result. + SetUnmatchedElements(_matchResult, _recognizedElements); + return _matchResult; } - enum RES { - MATCHED, - CASE_MISMATCH, - WHITESPACE_CORRECTED, - NO_MATCH - } + /** + * Find all unmatched elements and update the matcher result. + */ + private void SetUnmatchedElements(MatcherResult matcherResult, List recognizedElements) { + Set removeRecognizedList = new HashSet<>(); + matcherResult.getLocationMatches().forEach(it -> removeRecognizedList.addAll(it.recognizedElements)); - class Test { - char expected; - char found; - RES matchRes; + // Remove the elements which have been matched with expected elements. + removeRecognizedList.forEach(recognizedElements::remove); - public Test(char e) { - expected = e; - found = Character.MIN_VALUE; - matchRes = RES.NO_MATCH; - } + // The remainder of the recognized elements can be considered to be unmatched. + recognizedElements.forEach(matcherResult::addNoLocationMatch); - public String toString() { - return "Expected :" + expected + " Found :" + found + " Matched :" + matchRes; - } + Logger.log(Level.INFO, TAG, "No location match for the following detected text elements:\n {}", matcherResult.getNoLocationMatches().stream() + .map(e -> e._text + " " + e._location + "\n") + .collect(Collectors.joining())); } - private void analyze(Match match) { - String expectedText = match.expectedElement._text; - Logger.log(Level.INFO, TAG, "Trying to match {}", expectedText); - // Try to sort the found elements based on their location, - // map them to a grid based on coordinates starting with Upper left 0.0 - List sorted = match.recognizedElements.stream() - .sorted(Comparator.comparingInt(o -> o._location.x)) - .collect(Collectors.toList()); - - // Create list of Found chars and matched FLAG - List> unMatched = sorted.stream().flatMap( - recognizedElement -> recognizedElement._text.chars().boxed().map(it -> Character.toChars(it)[0])) - .map(it -> Pair.from(it, false)) - .collect(Collectors.toList()); - - - // Try to make the longest possible match - // For each char that matches the expected text mark as matched and move one ahead. - - // If not equal, could it be a case mismatch? - // if they are not equal at all - - - List result = new ArrayList<>(); - int unMatchedSize = unMatched.size(); - int indexCounter = 0; - for (int i = 0; i < expectedText.length(); i++) { - Character actual = expectedText.charAt(i); - Test charRes = new Test(actual); - - // Iterate over the - for (int k = indexCounter; k < unMatchedSize; k++) { - Pair item = unMatched.get(k); - Character found = item.left(); - // Try to match the actual char case sensitive: - if (found == actual && !item.right()) { - charRes.found = found; - unMatched.set(k, Pair.from(found, true)); - charRes.matchRes = RES.MATCHED; - // We have found a match inside the unMatched List move one position. - indexCounter = k + 1; - break; - } - - if (Character.isLetter(actual)) { - // Try to match the actual char none sensitive: - int CASING = 32; - if (Math.abs(found.compareTo(actual)) == CASING && !item.right()) { - charRes.found = found; - unMatched.set(k, Pair.from(found, true)); - charRes.matchRes = RES.CASE_MISMATCH; - // We have found a match inside the unMatched List move one position. - indexCounter = k + 1; - break; - } - } - } - - result.add(i, charRes); - } - - // Try to auto correct the missing whitespace - // Skip the first and last since we can't match left and right element - // TODO TM: Should we try to convert all the white spaces - int almostLast = result.size() - 1; - for (int i = 1; i < almostLast; i++) { - - Test res = result.get(i); - if (res.matchRes == RES.NO_MATCH && Character.isWhitespace(res.expected)){ - // Found a candidate, try to fix it - if (result.get(i-1).matchRes != RES.NO_MATCH && result.get(i+1).matchRes != RES.NO_MATCH ){ - // Surroundings are matched this must be a missing whitespace not detected by OCR - res.matchRes = RES.WHITESPACE_CORRECTED; - result.set(i, res); - Logger.log(Level.INFO, TAG, "Auto corrected whitespace in between expected text"); + /** + * Find all recognized text elements which intersect with the expected text element. + */ + @Nullable + private LocationMatch getIntersectedElements(List recognizedElements, ExpectedElement expectedElement) { + final LocationMatch[] locationMatches = {null}; + recognizedElements.forEach(it -> { + final int margin = config.locationMatchMargin; + Rectangle areaOfInterest = it._location; + areaOfInterest.setBounds( + areaOfInterest.x - margin, + areaOfInterest.y - margin, + areaOfInterest.width + (2 * margin), + areaOfInterest.height + (2 * margin) + ); + + // If the recognized element lay inside the area of interest. + if (areaOfInterest.intersects(expectedElement._location)) { + // Prepare to make a match. + if (locationMatches[0] == null) { + locationMatches[0] = new LocationMatch(expectedElement, margin); } + locationMatches[0].addRecognizedElement(it); } - } - - result.forEach(i -> Logger.log(Level.INFO, TAG, "{}", i)); + }); + return locationMatches[0]; } } - -//TODO see how to deal with reconginized elements which are placed/assinged to incorrect text fields diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java new file mode 100644 index 000000000..a5079119f --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java @@ -0,0 +1,33 @@ +package nl.ou.testar.visualvalidation.matcher; + +import org.testar.settings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class MatcherConfiguration extends ExtendedSettingBase { + public Integer locationMatchMargin; + + @Override + public String toString() { + return "OcrConfiguration{" + + "margin=" + locationMatchMargin + + '}'; + } + + public static MatcherConfiguration CreateDefault() { + MatcherConfiguration instance = new MatcherConfiguration(); + instance.locationMatchMargin = 0; + return instance; + } + + @Override + public int compareTo(MatcherConfiguration other) { + int result = -1; + if (locationMatchMargin.equals(other.locationMatchMargin)) { + result = 0; + } + return result; + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java index 6f4fbb1e2..bee71b376 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java @@ -7,29 +7,39 @@ public class MatcherResult { private final Set noMatches = new HashSet<>(); - private final Set matches = new HashSet<>(); + private final Set locationMatches = new HashSet<>(); + private final Set contentMatchResults = new HashSet<>(); - public void addMatch(Match match) { - matches.add(match); + public void addContentMatchResult(ContentMatchResult result) { + contentMatchResults.add(result); } - public void addNoMatch(TextElement element) { + public void addLocationMatch(LocationMatch locationMatch) { + locationMatches.add(locationMatch); + } + + public void addNoLocationMatch(TextElement element) { noMatches.add(element); } - public Set getNoMatches() { + public Set getNoLocationMatches() { return noMatches; } - public Set getMatches() { - return matches; + public Set getLocationMatches() { + return locationMatches; + } + + public Set getResult() { + return contentMatchResults; } @Override public String toString() { return "MatcherResult{" + - "noMatches(" + noMatches.size() + ")=" + noMatches + - ", matches(" + matches.size() + ")=" + matches + + "noLocationMatches(" + noMatches.size() + ")=" + noMatches + + ", locationMatches(" + locationMatches.size() + ")=" + locationMatches + + ", contentMatches(" + contentMatchResults.size() + ")=" + contentMatchResults + '}'; } } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java index b1224ad66..61ec20aac 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java @@ -5,7 +5,7 @@ public static VisualMatcher createDummyMatcher() { return new VisualDummyMatcher(); } - public static VisualMatcher createLocationMatcher() { - return new LocationMatcher(); + public static VisualMatcher createLocationMatcher(MatcherConfiguration setting) { + return new LocationMatcher(setting); } } From 5c3bfc13fbbb1839d10bd7cf5ee62284cd48a229 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 23 Jun 2021 21:41:47 +0200 Subject: [PATCH 23/40] Fixed rebase issues. --- testar/src/org/fruit/monkey/Settings.java | 7 - .../src/org/fruit/monkey/SettingsDialog.java | 3 +- .../org/testar/settings/ExtendedSettings.java | 271 ------------------ .../testar/settings/ExtendedSettingsTest.java | 238 --------------- 4 files changed, 2 insertions(+), 517 deletions(-) delete mode 100644 testar/src/org/testar/settings/ExtendedSettings.java delete mode 100644 testar/test/org/testar/settings/ExtendedSettingsTest.java diff --git a/testar/src/org/fruit/monkey/Settings.java b/testar/src/org/fruit/monkey/Settings.java index 3924c6403..440ac29f1 100644 --- a/testar/src/org/fruit/monkey/Settings.java +++ b/testar/src/org/fruit/monkey/Settings.java @@ -443,13 +443,6 @@ public String toFileString() throws IOException{ +"ExtendedSettingsFile =" + Util.lineSep() +"\n" +"#################################################################\n" - +"# Extended settings file\n" - +"#\n" - +"# Relative path to extended settings file.\n" - +"#################################################################\n" - +"ExtendedSettingsFile =" + Util.lineSep() - +"\n" - +"#################################################################\n" +"# Other more advanced settings\n" +"#################################################################\n"); diff --git a/testar/src/org/fruit/monkey/SettingsDialog.java b/testar/src/org/fruit/monkey/SettingsDialog.java index a5392c670..8363149af 100644 --- a/testar/src/org/fruit/monkey/SettingsDialog.java +++ b/testar/src/org/fruit/monkey/SettingsDialog.java @@ -38,6 +38,7 @@ import org.fruit.Util; import org.fruit.alayer.exceptions.NoSuchTagException; import org.fruit.monkey.dialog.*; +import org.testar.settings.ExtendedSettingFile; import org.testar.settings.ExtendedSettingsFactory; import javax.imageio.ImageIO; @@ -190,7 +191,7 @@ private void checkSettings(Settings settings) throws IllegalStateException { try{ settings.get(ConfigTags.ExtendedSettingsFile); } catch (NoSuchTagException e){ - settings.set(ConfigTags.ExtendedSettingsFile, settingsFile.replace(SETTINGS_FILENAME, ExtendedSettings.FileName)); + settings.set(ConfigTags.ExtendedSettingsFile, settingsFile.replace(SETTINGS_FILENAME, ExtendedSettingFile.FileName)); } settingPanels.forEach((k,v) -> v.right().checkSettings()); diff --git a/testar/src/org/testar/settings/ExtendedSettings.java b/testar/src/org/testar/settings/ExtendedSettings.java deleted file mode 100644 index 506e454eb..000000000 --- a/testar/src/org/testar/settings/ExtendedSettings.java +++ /dev/null @@ -1,271 +0,0 @@ -package org.testar.settings; - -import org.apache.commons.lang.ObjectUtils; -import org.apache.commons.lang.SerializationUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.checkerframework.checker.nullness.qual.NonNull; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAnyElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * Generic XML root element. - */ -@XmlRootElement(name = "root") -@XmlAccessorType(XmlAccessType.FIELD) -class ExtendedSettingsXml implements Serializable { - // Holds all the XML elements found in the file. - @XmlAnyElement(lax = true) - public List any; - - public Integer version; - - public ExtendedSettingsXml() { - any = new ArrayList<>(); - version = 1; - } -} - -/** - * Helper class for reading XML data from disk. - */ -class ExtractionResult { - final JAXBContext Context; - final ExtendedSettingsXml Data; - final Boolean FileNotFound; - - public ExtractionResult(JAXBContext context, ExtendedSettingsXml data, Boolean fileNotFound) { - Context = context; - Data = data; - FileNotFound = fileNotFound; - } -} - -public class ExtendedSettings implements Serializable { - private static final Logger LOGGER = LogManager.getLogger(); - public static final String FileName = "ExtendedSettings.xml"; - private final String _absolutePath; - private final ReentrantReadWriteLock _fileAccessMutex; - - /** - * Stores a deep copy of the object up on reading. The {@link #load(Class)} returns a reference to the XML element. - * If we modify that reference and want to save it, we need to replace the old value. We use {@code #_loadedValue} - * to find and update the tag in the XML file. - */ - private Object _loadedValue = null; - - /** - * Constructor, each specialization must have a unique implementation of this class. - * - * @param fileLocation The absolute path the the XML file. - * @param fileAccessMutex Mutex for thread-safe access. - */ - protected ExtendedSettings(@NonNull String fileLocation, @NonNull ReentrantReadWriteLock fileAccessMutex) { - _fileAccessMutex = fileAccessMutex; - _absolutePath = System.getProperty("user.dir") + - (fileLocation.startsWith(".") ? fileLocation.substring(1) : (fileLocation.startsWith(File.separator) - ? fileLocation : File.separator + fileLocation)); - } - - /** - * Try to load the requested data element from the XML file. - * - * @param clazz The class type of the element we want to load. - * @param The class type of the element we want to load. - * @return When found in the XML the requested element, otherwise null. - */ - @SuppressWarnings("unchecked") - public T load(@SuppressWarnings("rawtypes") @NonNull Class clazz) { - T result = null; - - // Check if file exits - ExtractionResult rd = readFile(clazz); - - if (Objects.nonNull(rd.Data)) { - // Try to find the section of interest. - result = (T) rd.Data.any.stream() - .filter(element -> element.getClass() == clazz) - .findFirst() - .orElse(null); - - // We only support loading a single element for now. - if (rd.Data.any.stream().filter(element -> element.getClass() == clazz).count() > 1) { - LOGGER.error("Duplicate elements found for {}, returning first element ", clazz); - } - - // Store the current content, so we can replace it when needed. - _loadedValue = SerializationUtils.clone((Serializable) result); - } - if (result == null) { - LOGGER.info("Did not found XML element for class: {}", clazz); - } - - return result; - } - - /** - * Try to load the requested data element from the XML file. - * If not found, the default configuration is written to disk and returned. - * - * @param clazz The class type of the element we want to load. - * @param defaultFunctor The function to create the default configuration for class #clazz. - * @param The class type of the element we want to load. - * @return Either the element found in the file otherwise the default configuration. - */ - public T load(@SuppressWarnings("rawtypes") @NonNull Class clazz, @NonNull IExtendedSettingDefaultValue defaultFunctor) { - T result = load(clazz); - - if (result == null) { - LOGGER.info("Writing default values for {}", clazz); - save(defaultFunctor.CreateDefault()); - return load(clazz); - } - - return result; - } - - /** - * Save the data to the XML file. - * The file is created if it does not exist. - * - * @param data The data we need to store. - */ - public void save(@NonNull Object data) { - if (!(data instanceof Comparable)) { - LOGGER.error("Object {} is not extending Comparable", data); - return; - } - - ExtractionResult result = readFile(data.getClass()); - updateFile(data, result.Context, result.Data); - } - - @SuppressWarnings("rawtypes") - private ExtractionResult extractContent(@NonNull Class clazz) { - JAXBContext context = null; - ExtendedSettingsXml data = null; - boolean fileNotFound = false; - - try { - FileInputStream xmlFile = null; - context = JAXBContext.newInstance(ExtendedSettingsXml.class, clazz); - Unmarshaller um = context.createUnmarshaller(); - - try { - _fileAccessMutex.readLock().lock(); - // Load latest version of XML, other settings may have been updated in the mean time. - xmlFile = new FileInputStream(_absolutePath); - data = (ExtendedSettingsXml) um.unmarshal(xmlFile); - } catch (FileNotFoundException e) { - fileNotFound = true; - } finally { - if (Objects.nonNull(xmlFile)) { - xmlFile.close(); - } - } - } catch (IOException | JAXBException e) { - e.printStackTrace(); - } finally { - _fileAccessMutex.readLock().unlock(); - } - return new ExtractionResult(context, data, fileNotFound); - } - - @SuppressWarnings("rawtypes") - private ExtractionResult readFile(@NonNull Class clazz) { - - ExtractionResult result = extractContent(clazz); - if (result.FileNotFound) { - createExtendedSettingsFile(); - result = extractContent(clazz); - } - - return result; - } - - @SuppressWarnings("rawtypes") - private void updateFile(@NonNull Object data, JAXBContext context, ExtendedSettingsXml xmlData) { - Objects.requireNonNull(context); - Objects.requireNonNull(xmlData); - - try { - _fileAccessMutex.writeLock().lock(); - OutputStream os = null; - try { - os = new FileOutputStream(_absolutePath); - Marshaller marshaller = context.createMarshaller(); - marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); - - // Try to find the old XML tag. - Object found = null; - for (Object element : xmlData.any) { - if (element.getClass() == data.getClass()) { - if (ObjectUtils.compare((Comparable) element, (Comparable) _loadedValue) == 0) { - found = element; - } - } - } - - // Replace old content with new. - xmlData.any.remove(found); - xmlData.any.add(data); - - // Update the file. - marshaller.marshal(xmlData, os); - } catch (JAXBException e) { - e.printStackTrace(); - } finally { - if (Objects.nonNull(os)) { - os.close(); - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - _fileAccessMutex.writeLock().unlock(); - } - } - - private void createExtendedSettingsFile() { - try { - _fileAccessMutex.writeLock().lock(); - OutputStream os = null; - try { - os = new FileOutputStream(_absolutePath); - JAXBContext context = JAXBContext.newInstance(ExtendedSettingsXml.class); - Marshaller marshaller = context.createMarshaller(); - marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); - marshaller.marshal(new ExtendedSettingsXml(), os); - LOGGER.info("Created extended settings file: {}", _absolutePath); - } catch (JAXBException e) { - e.printStackTrace(); - } finally { - if (Objects.nonNull(os)) { - os.close(); - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - _fileAccessMutex.writeLock().unlock(); - } - } -} diff --git a/testar/test/org/testar/settings/ExtendedSettingsTest.java b/testar/test/org/testar/settings/ExtendedSettingsTest.java deleted file mode 100644 index c610b66da..000000000 --- a/testar/test/org/testar/settings/ExtendedSettingsTest.java +++ /dev/null @@ -1,238 +0,0 @@ -package org.testar.settings; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.FileNotFoundException; -import java.util.Arrays; -import java.util.Objects; -import java.util.Scanner; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.junit.Assert.*; - -public class ExtendedSettingsTest { - final private String _rootDir = System.getProperty("user.dir") + File.separator; - final private String _relativePath = "extended_settings_test" + File.separatorChar; - final private String _workingDir = _rootDir + _relativePath; - - ExtendedSettings sut; - ReentrantReadWriteLock fileAccessLock; - - @After - public void CleanUp() { - removeAllXmlTestFiles(); - } - - @Before - public void Setup() { - File directory = new File(_workingDir); - if (!directory.exists()) { - assertTrue(directory.mkdir()); - } - removeAllXmlTestFiles(); - - fileAccessLock = new ReentrantReadWriteLock(); - } - - void removeAllXmlTestFiles() { - // Search and delete all XML files - for (File file : Objects.requireNonNull(new File(_workingDir).listFiles())) { - if (file.getName().endsWith(".xml")) { - assertTrue(file.delete()); - } - } - } - - Boolean fileContains(String absolutePath, String... expectedLines) { - Boolean[] result = new Boolean[expectedLines.length]; - Arrays.fill(result, Boolean.FALSE); - - try { - File file = new File(absolutePath); - for (int i = 0; i < expectedLines.length; i++) { - Scanner myReader = new Scanner(file); - while (myReader.hasNextLine()) { - String data = myReader.nextLine(); - if (data.replaceAll("\\s+", "") - .equalsIgnoreCase(expectedLines[i].replaceAll("\\s+", ""))) { - result[i] = true; - break; - } - } - myReader.close(); - } - } catch (FileNotFoundException e) { - System.out.println("An error occurred."); - e.printStackTrace(); - } - - return Arrays.stream(result).allMatch(x -> x); - } - - @Test - public void settingFileCreationWhenNotExisting() { - // GIVEN The file doesn't exist. - final String unknownFile = "ghost.xml"; - File testFile = new File(_workingDir + unknownFile); - assertFalse(testFile.exists()); - - // WHEN Trying to load an extended setting without default values. - sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); - TestSetting element = sut.load(TestSetting.class); - - // THEN The file is created. - assertTrue(testFile.exists()); - assertNull(element); - } - - @Test - public void settingFileCreationWithDefaultValuesWhenNotExisting() { - // GIVEN The file doesn't exist. - final String unknownFile = "default.xml"; - File testFile = new File(_workingDir + unknownFile); - assertFalse(testFile.exists()); - - // WHEN Trying to load an extended setting without default values. - sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); - TestSetting element = sut.load(TestSetting.class, TestSetting::CreateDefault); - - // THEN The file is created containing default values. - assertTrue(testFile.exists()); - assertNotNull(element); - assertEquals(TestSetting.DEFAULT_VALUE, element.value); - } - - @Test() - public void saveValueWhenFileNotExists() { - // GIVEN The file doesn't exist. - final String unknownFile = "save.xml"; - File testFile = new File(_workingDir + unknownFile); - assertFalse(testFile.exists()); - - // WHEN Trying to save an extended setting before loading. - sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); - TestSetting element = new TestSetting(); - sut.save(element); - - // THEN The file is created. - assertTrue(testFile.exists()); - } - - @Test - public void loadFromFileWithOnlyUnknownElements() { - // GIVEN The file contains only unknown elements. - final String unknownFile = "unknown.xml"; - XmlFile.CreateUnknownFile(_workingDir + unknownFile); - - // WHEN Trying to load an extended setting without default values. - sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); - TestSetting element = sut.load(TestSetting.class); - - // THEN The element remains empty. - assertNull(element); - } - - @Test - public void saveToFileWithOnlyUnknownElements() { - // GIVEN The file contains only unknown elements. - final String unknownFile = "unknown_save.xml"; - XmlFile.CreateUnknownFile(_workingDir + unknownFile); - - // WHEN Trying to save an extended setting. - sut = new ExtendedSettings(_relativePath + unknownFile, fileAccessLock); - TestSetting data = TestSetting.CreateDefault(); - sut.save(data); - - // THEN The data is added to the file. - assertTrue(fileContains(_workingDir + unknownFile, "", - "Default", - "")); - } - - @Test - public void updateKnownElement() { - // GIVEN The file contains one known element. - final String knownFile = "update_known.xml"; - XmlFile.CreateSingleTestSetting(_workingDir + knownFile); - - // WHEN Trying to update a known element. - sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); - TestSetting data = sut.load(TestSetting.class); - data.value = "updated"; - sut.save(data); - - // THEN The data is updated and saved to the file. - assertTrue(fileContains(_workingDir + knownFile, "", - "updated", - "")); - } - - @Test - public void updateKnownElementWhenFileMultipleKnownElements() { - // GIVEN The file contains multiple known elements. - final String knownFile = "update_known_multiple.xml"; - XmlFile.CreateMultipleTestSetting(_workingDir + knownFile); - - // WHEN Trying to update a known element. - sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); - TestSetting data = sut.load(TestSetting.class); - data.value = "version3"; - sut.save(data); - - // THEN The first element is updated and saved to the file. - assertTrue(fileContains(_workingDir + knownFile, "version3", "version2")); - assertFalse(fileContains(_workingDir + knownFile, "version1")); - } - - @Test - public void updateMultipleElement() { - // GIVEN The file contains multiple known elements. - final String knownFile = "update_multiple_elements.xml"; - File testFile = new File(_workingDir + knownFile); - assertFalse(testFile.exists()); - sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); - ExtendedSettings sutTwo = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); - TestSetting elementOne = sut.load(TestSetting.class, TestSetting::CreateDefault); - OtherSetting elementTwo = sutTwo.load(OtherSetting.class, OtherSetting::CreateDefault); - assertTrue(testFile.exists()); - assertNotNull(elementOne); - - // WHEN Trying to update the first element. - elementOne.value = "updated"; - sut.save(elementOne); - - // THEN The update is stored. - assertEquals("updated", elementOne.value); - assertTrue(fileContains(_workingDir + knownFile, "updated", "5")); - - // WHEN Trying to update the second element. - elementTwo.speed = 6; - sutTwo.save(elementTwo); - - // THEN The update is stored. - assertEquals(6, elementTwo.speed); - assertTrue(fileContains(_workingDir + knownFile, "updated", "6")); - } - - @Test(expected = ClassCastException.class) - public void updateElementWithWrongBaseClass() { - // GIVEN The file contains multiple known elements. - final String knownFile = "update_wrong_element.xml"; - File testFile = new File(_workingDir + knownFile); - assertFalse(testFile.exists()); - sut = new ExtendedSettings(_relativePath + knownFile, fileAccessLock); - TestSetting elementOne = sut.load(TestSetting.class, TestSetting::CreateDefault); - OtherSetting elementTwo = sut.load(OtherSetting.class, OtherSetting::CreateDefault); - assertTrue(testFile.exists()); - assertNotNull(elementOne); - - // WHEN Trying to update the first element while the _loadedvalue has been assigned to OtherSetting. - elementOne.value = "updated"; - sut.save(elementOne); - - // THEN An exception should been throw. - } -} From a7595fd3ea2e3a27cec8f5f12699103ab1aa150c Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Thu, 24 Jun 2021 22:24:52 +0200 Subject: [PATCH 24/40] Added unit tests. --- testar/build.gradle | 2 + .../matcher/ContentMatcher.java | 8 +- .../matcher/LocationMatcher.java | 6 +- .../matcher/ContentMatcherTest.java | 153 ++++++++++++++++++ .../matcher/LocationMatcherTest.java | 130 +++++++++++++++ 5 files changed, 292 insertions(+), 7 deletions(-) create mode 100644 testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java create mode 100644 testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java diff --git a/testar/build.gradle b/testar/build.gradle index 95444c357..16bbb18e3 100644 --- a/testar/build.gradle +++ b/testar/build.gradle @@ -92,6 +92,8 @@ dependencies { compile group: 'org.bytedeco', name: 'javacv', version: '1.5.5' compile group: 'commons-io', name: 'commons-io', version: '2.7' runtime project(':windows') + + testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19' } evaluationDependsOn(':windows') diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java index a52183f57..d5dda3ffa 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java @@ -14,7 +14,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -public class ContentMatcher { +class ContentMatcher { static final String TAG = "ContentMatcher"; /** @@ -73,7 +73,7 @@ public static ContentMatchResult Match(LocationMatch locationMatch) { } @NonNull - private static List sortRecognizedElements(LocationMatch locationMatch) { + static List sortRecognizedElements(LocationMatch locationMatch) { Map> lineBucket = sortRecognizedElementsPerTextLine(locationMatch); // Get the sorted Y-axe coordinates for the identified lines. @@ -93,7 +93,7 @@ private static List sortRecognizedElements(LocationMatch loca } @NonNull - private static Map> sortRecognizedElementsPerTextLine(LocationMatch locationMatch) { + static Map> sortRecognizedElementsPerTextLine(LocationMatch locationMatch) { // Determine the average height of the recognized text elements. double lineHeight = locationMatch.recognizedElements.stream() .mapToInt(e -> e._location.height) @@ -132,7 +132,7 @@ private static Map> sortRecognizedElementsPerTe return lineBucket; } - private static void correctWhitespaces(ExpectedTextMatchResult expectedResult) { + static void correctWhitespaces(ExpectedTextMatchResult expectedResult) { // Whitespaces are automatically corrected. for (CharacterMatch res : expectedResult.expectedText) { if (res.result == CharacterMatchResult.NO_MATCH && Character.isWhitespace(res.character.character)) { diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java index 4c166cc86..232a51dc8 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java @@ -69,14 +69,14 @@ public MatcherResult Match(List recognizedElements, List recognizedElements) { + void setUnmatchedElements(MatcherResult matcherResult, List recognizedElements) { Set removeRecognizedList = new HashSet<>(); matcherResult.getLocationMatches().forEach(it -> removeRecognizedList.addAll(it.recognizedElements)); @@ -95,7 +95,7 @@ private void SetUnmatchedElements(MatcherResult matcherResult, List recognizedElements, ExpectedElement expectedElement) { + LocationMatch getIntersectedElements(List recognizedElements, ExpectedElement expectedElement) { final LocationMatch[] locationMatches = {null}; recognizedElements.forEach(it -> { final int margin = config.locationMatchMargin; diff --git a/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java b/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java new file mode 100644 index 000000000..30068e7c7 --- /dev/null +++ b/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java @@ -0,0 +1,153 @@ +package nl.ou.testar.visualvalidation.matcher; + +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; +import org.junit.Test; + +import java.awt.Rectangle; +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.*; + +public class ContentMatcherTest { + + private static final RecognizedElement firstLine = new RecognizedElement(new Rectangle(0, 0, 20, 10), 100, "1"); + private static final RecognizedElement secondLine = new RecognizedElement(new Rectangle(0, 10, 20, 10), 100, "2W"); + private static final RecognizedElement thirdLineFirst = new RecognizedElement(new Rectangle(0, 20, 20, 10), 100, "3 "); + private static final RecognizedElement thirdLineSecond = new RecognizedElement(new Rectangle(10, 25, 20, 5), 100, "?"); + private static final ExpectedElement expectedElement = new ExpectedElement(new Rectangle(0, 0, 20, 30), "1K\n\r2w\n\r3"); + + private LocationMatch prepareExpectedTextWith3Lines() { + LocationMatch locationMatch = new LocationMatch(expectedElement, 0); + // Shuffled input so we can test the algorithm. + locationMatch.addRecognizedElement(firstLine); + locationMatch.addRecognizedElement(thirdLineFirst); + locationMatch.addRecognizedElement(secondLine); + locationMatch.addRecognizedElement(thirdLineSecond); + + return locationMatch; + } + + @Test + public void match() { + // GIVEN we have an expected text containing three text lines. + LocationMatch input = prepareExpectedTextWith3Lines(); + + // WHEN we run the algorithm. + ContentMatchResult result = ContentMatcher.Match(input); + + // THEN the result must be: + assertEquals(9, result.totalExpected); + assertEquals(8, result.totalMatched); + + assertSame('1', result.expectedResult.expectedText.get(0).character.character); + assertEquals(CharacterMatchResult.MATCHED, result.expectedResult.expectedText.get(0).result); + assertSame('1', result.expectedResult.expectedText.get(0).character.match.character); + + assertSame('K', result.expectedResult.expectedText.get(1).character.character); + assertEquals(CharacterMatchResult.NO_MATCH, result.expectedResult.expectedText.get(1).result); + assertNull(result.expectedResult.expectedText.get(1).character.match); + + assertSame('\n', result.expectedResult.expectedText.get(2).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(2).result); + assertNull(result.expectedResult.expectedText.get(1).character.match); + + assertSame('\r', result.expectedResult.expectedText.get(3).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(3).result); + assertNull(result.expectedResult.expectedText.get(3).character.match); + + assertSame('2', result.expectedResult.expectedText.get(4).character.character); + assertEquals(CharacterMatchResult.MATCHED, result.expectedResult.expectedText.get(4).result); + assertSame('2', result.expectedResult.expectedText.get(4).character.match.character); + + assertSame('w', result.expectedResult.expectedText.get(5).character.character); + assertEquals(CharacterMatchResult.CASE_MISMATCH, result.expectedResult.expectedText.get(5).result); + assertSame('W', result.expectedResult.expectedText.get(5).character.match.character); + + assertSame('\n', result.expectedResult.expectedText.get(6).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(6).result); + assertNull(result.expectedResult.expectedText.get(6).character.match); + + assertSame('\r', result.expectedResult.expectedText.get(7).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(7).result); + assertNull(result.expectedResult.expectedText.get(7).character.match); + + assertSame('3', result.expectedResult.expectedText.get(8).character.character); + assertEquals(CharacterMatchResult.MATCHED, result.expectedResult.expectedText.get(8).result); + assertSame('3', result.expectedResult.expectedText.get(8).character.match.character); + + assertSame('?', result.recognizedResult.recognized.get(5).character); + assertNull(result.recognizedResult.recognized.get(5).match); + } + + @Test + public void sortRecognizedElements() { + // GIVEN we have an expected text containing three text lines. + LocationMatch input = prepareExpectedTextWith3Lines(); + + // WHEN we sort the recognized elements. + List result = ContentMatcher.sortRecognizedElements(input); + + // THEN the recognized elements should be sorted correctly. + assertEquals(Arrays.asList(firstLine, secondLine, thirdLineFirst, thirdLineSecond), result); + } + + @Test + public void sortRecognizedElementsPerTextLine() { + // GIVEN we have an expected text containing three text lines. + LocationMatch input = prepareExpectedTextWith3Lines(); + + // WHEN we sort the recognized elements only on their y coordinate. + Map> result = ContentMatcher.sortRecognizedElementsPerTextLine(input); + + // THEN the recognized elements should be sorted correctly. + Map> expectedResult = Stream.of( + new AbstractMap.SimpleEntry<>(5, Collections.singletonList(firstLine)), + new AbstractMap.SimpleEntry<>(15, Collections.singletonList(secondLine)), + new AbstractMap.SimpleEntry<>(25, Arrays.asList(thirdLineFirst, thirdLineSecond))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + expectedResult.keySet().forEach(expectedKey -> + { + assertEquals(expectedResult.get(expectedKey).size(), result.get(expectedKey).size()); + expectedResult.get(expectedKey).forEach(it -> assertTrue(result.get(expectedKey).contains(it))); + } + ); + } + + @Test + public void correctAllNoneMatchedWhitespaces() { + // GIVEN a prepared result containing whitespace characters which are not marked as matched. + ExpectedTextMatchResult input = new ExpectedTextMatchResult(" T\te\fs\nt\r.\u000B?"); + Arrays.asList(1, 3, 5, 7, 9, 11).forEach(it -> + input.expectedText.get(it).result = CharacterMatchResult.MATCHED + ); + + // WHEN the correction mechanism is applied + ContentMatcher.correctWhitespaces(input); + + // THEN all the whitespaces should be corrected. + for (int i = 0; i < input.expectedText.size(); i++) { + assertSame(i % 2 == 0 ? CharacterMatchResult.WHITESPACE_CORRECTED : CharacterMatchResult.MATCHED, + input.expectedText.get(i).result); + } + } + + @Test + public void notCorrectMatchedWhitespaces() { + // GIVEN a prepared result containing whitespace characters which are matched. + ExpectedTextMatchResult input = new ExpectedTextMatchResult(" T"); + input.expectedText.forEach(it -> it.result = CharacterMatchResult.MATCHED); + + // WHEN the correction mechanism is applied + ContentMatcher.correctWhitespaces(input); + + // THEN all the whitespaces should be corrected. + input.expectedText.forEach(it -> assertSame(CharacterMatchResult.MATCHED, it.result)); + } +} diff --git a/testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java b/testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java new file mode 100644 index 000000000..c165addde --- /dev/null +++ b/testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java @@ -0,0 +1,130 @@ +package nl.ou.testar.visualvalidation.matcher; + +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; +import nl.ou.testar.visualvalidation.ocr.RecognizedElement; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.awt.Rectangle; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@RunWith(MockitoJUnitRunner.class) +public class LocationMatcherTest { + + private static final RecognizedElement recognizedElementOne = + new RecognizedElement(new Rectangle(0, 15, 36, 1), 0, "r"); + private static final RecognizedElement recognizedElementTwo = + new RecognizedElement(new Rectangle(40, 16, 26, 45), 23, "df"); + private static final RecognizedElement recognizedElementThree = + new RecognizedElement(new Rectangle(5, 10, 26, 45), 23, "aa"); + private static final RecognizedElement recognizedElementFour = + new RecognizedElement(new Rectangle(12, 16, 10, 20), 23, "Open"); + private static final ExpectedElement expectedElement = + new ExpectedElement(new Rectangle(10, 15, 15, 23), "open"); + + @Test + public void successfulMatch() { + // GIVEN we have one expected text element with a matching(case corrected for first character) + // recognized element plus three additional garbage elements. + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + List recognizedElements = Arrays.asList( + recognizedElementOne, recognizedElementTwo, + recognizedElementThree, recognizedElementFour); + + List expectedElements = Collections.singletonList(expectedElement); + + // WHEN we run the match algorithm + MatcherResult result = sut.Match(recognizedElements, expectedElements); + + // THEN The match result should be: + // We expected that one recognized elements isn't matched based on the location. + assertEquals(1, result.getNoLocationMatches().size()); + assertTrue(result.getNoLocationMatches().contains(recognizedElementTwo)); + + // We have 3 recognized elements that intersect with the expected text. + assertEquals(1, result.getLocationMatches().size()); + LocationMatch locationMatch = result.getLocationMatches().iterator().next(); + assertEquals(3, locationMatch.recognizedElements.size()); + + // Validate that all expected characters have been matched. + assertEquals(1, result.getResult().size()); + assertEquals(4, result.getResult().iterator().next().totalMatched); + } + + @Test + public void setUnmatchedElements() { + // GIVEN we have a matched only the first recognized element. + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + MatcherResult matcherResult = Mockito.mock(MatcherResult.class); + LocationMatch locationMatchMock = Mockito.mock(LocationMatch.class); + locationMatchMock.recognizedElements = new HashSet<>(Collections.singletonList(recognizedElementOne)); + Mockito.when(matcherResult.getLocationMatches()).thenReturn(new HashSet<>(Collections.singletonList(locationMatchMock))); + + ArrayList recognizedElements = new ArrayList<>(Arrays.asList( + recognizedElementOne, + recognizedElementTwo, + recognizedElementThree + )); + + // WHEN we collect the unmatched results. + sut.setUnmatchedElements(matcherResult, recognizedElements); + + // THEN the matcher result should contain the two remaining recognized elements. + ArgumentCaptor arg = ArgumentCaptor.forClass(RecognizedElement.class); + verify(matcherResult, times(2)).addNoLocationMatch(arg.capture()); + assertEquals(2, recognizedElements.size()); + assertEquals(arg.getAllValues().get(0), recognizedElementTwo); + assertEquals(arg.getAllValues().get(1), recognizedElementThree); + } + + @Test + public void getIntersectedElements() { + // GIVEN two recognized elements inside and one outside the surface area of the expected text. + List recognizedElements = Arrays.asList( + recognizedElementOne, + recognizedElementTwo, + recognizedElementThree + ); + + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + // WHEN we try find intersecting elements. + LocationMatch result = sut.getIntersectedElements(recognizedElements, expectedElement); + + // THEN the two recognized elements inside the area mus tbe listed in the outcome. + assertNotNull(result); + assertEquals(expectedElement._location, result.location.location); + assertEquals(new HashSet<>(Arrays.asList(recognizedElementOne, recognizedElementThree)), result.recognizedElements); + } + + @Test + public void getIntersectedElementsReturnsNull() { + // GIVEN recognized elements outside the surface area of the expected text. + List recognizedElements = Collections.singletonList(recognizedElementTwo); + + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + // WHEN we try find intersecting elements. + LocationMatch result = sut.getIntersectedElements(recognizedElements, expectedElement); + + // THEN none are expected. + assertNull(result); + } +} From a8fbb5906532226318150bf39b5d3859ecc89708 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 30 Jun 2021 15:18:04 +0200 Subject: [PATCH 25/40] Added visual validation support for action shots. Fixed action shot for eye & sikulix desktop protocol. --- core/src/es/upv/staq/testar/ProtocolUtil.java | 12 +-- ...rotocol_desktop_simple_stategraph_eye.java | 13 ++- ...col_desktop_simple_stategraph_sikulix.java | 11 +- .../DummyVisualValidator.java | 6 ++ .../VisualValidationManager.java | 9 ++ .../visualvalidation/VisualValidator.java | 12 ++- .../extractor/DummyExtractor.java | 6 +- .../extractor/ExpectedTextExtractorBase.java | 101 +++++++++--------- .../extractor/TextExtractorInterface.java | 4 +- .../src/org/fruit/monkey/DefaultProtocol.java | 12 ++- .../alayer/webdriver/WdProtocolUtil.java | 13 +-- 11 files changed, 121 insertions(+), 78 deletions(-) diff --git a/core/src/es/upv/staq/testar/ProtocolUtil.java b/core/src/es/upv/staq/testar/ProtocolUtil.java index 75e8b1e06..6184eae0c 100644 --- a/core/src/es/upv/staq/testar/ProtocolUtil.java +++ b/core/src/es/upv/staq/testar/ProtocolUtil.java @@ -260,8 +260,8 @@ public static AWTCanvas getStateshotBinary(State state) { AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(viewPort.x(), viewPort.y(), viewPort.width(), viewPort.height()), getRootWindowHandle(state), AWTCanvas.StorageFormat.PNG, 1); return scrshot; } - - public static String getActionshot(State state, Action action){ + + public static AWTCanvas getActionshot(State state, Action action){ List targets = action.get(Tags.Targets, null); if (targets != null){ Widget w; @@ -274,14 +274,14 @@ public static String getActionshot(State state, Action action){ r = new Rectangle((int)s.x(), (int)s.y(), (int)s.width(), (int)s.height()); actionArea = actionArea.union(r); } - if (actionArea.isEmpty()) + if (actionArea.isEmpty()) { return null; - AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(actionArea.x, actionArea.y, actionArea.width, actionArea.height), getRootWindowHandle(state), + } + return AWTCanvas.fromScreenshot(Rect.from(actionArea.x, actionArea.y, actionArea.width, actionArea.height), getRootWindowHandle(state), AWTCanvas.StorageFormat.PNG, 1); - return ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), scrshot); } return null; - } + } private static long getRootWindowHandle(State state) { long windowHandle = 0; diff --git a/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java b/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java index ba4130bc5..6bb41769d 100644 --- a/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java +++ b/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java @@ -34,9 +34,11 @@ import java.util.Set; import es.upv.staq.testar.ProtocolUtil; +import es.upv.staq.testar.serialisation.ScreenshotSerialiser; import nl.ou.testar.SimpleGuiStateGraph.GuiStateGraphWithVisitedActions; import nl.ou.testar.HtmlReporting.HtmlSequenceReport; import org.fruit.Util; +import org.fruit.alayer.AWTCanvas; import org.fruit.alayer.Action; import org.fruit.alayer.exceptions.*; import org.fruit.alayer.SUT; @@ -161,7 +163,10 @@ protected boolean executeAction(SUT system, State state, Action action){ //System.out.println("DEBUG: action: "+action.toString()); //System.out.println("DEBUG: action short: "+action.toShortString()); if(action.toShortString().equalsIgnoreCase("LeftClickAt")){ - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Eye eye = new Eye(); try { //System.out.println("DEBUG: sikuli clicking "); @@ -180,8 +185,10 @@ protected boolean executeAction(SUT system, State state, Action action){ }else if(action.toShortString().contains("ClickTypeInto(")){ String textToType = action.toShortString().substring(action.toShortString().indexOf("("), action.toShortString().indexOf(")")); //System.out.println("parsed text:"+textToType); - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); - Util.pause(halfWait); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Util.pause(halfWait); Eye eye = new Eye(); try { //System.out.println("DEBUG: sikuli typing "); diff --git a/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java b/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java index 66da10412..0c41ad07a 100644 --- a/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java +++ b/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java @@ -33,6 +33,7 @@ import java.util.Set; import es.upv.staq.testar.ProtocolUtil; +import es.upv.staq.testar.serialisation.ScreenshotSerialiser; import nl.ou.testar.SimpleGuiStateGraph.GuiStateGraphWithVisitedActions; import nl.ou.testar.HtmlReporting.HtmlSequenceReport; import org.fruit.Util; @@ -158,7 +159,10 @@ protected boolean executeAction(SUT system, State state, Action action){ //System.out.println("DEBUG: action: "+action.toString()); //System.out.println("DEBUG: action short: "+action.toShortString()); if(action.toShortString().equalsIgnoreCase("LeftClickAt")){ - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Screen sikuliScreen = new Screen(); try { //System.out.println("DEBUG: sikuli clicking "); @@ -175,7 +179,10 @@ protected boolean executeAction(SUT system, State state, Action action){ }else if(action.toShortString().contains("ClickTypeInto(")){ String textToType = action.toShortString().substring(action.toShortString().indexOf("("), action.toShortString().indexOf(")")); //System.out.println("parsed text:"+textToType); - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Util.pause(halfWait); Screen sikuliScreen = new Screen(); try { diff --git a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java index b3d77e3af..37e65ab45 100644 --- a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java @@ -3,6 +3,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.AWTCanvas; import org.fruit.alayer.State; +import org.fruit.alayer.Widget; public class DummyVisualValidator implements VisualValidationManager { @Override @@ -10,6 +11,11 @@ public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { } + @Override + public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { + + } + @Override public void Destroy() { diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java index e5eb6a821..1b325b078 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java @@ -3,6 +3,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.AWTCanvas; import org.fruit.alayer.State; +import org.fruit.alayer.Widget; public interface VisualValidationManager { /** @@ -12,6 +13,14 @@ public interface VisualValidationManager { */ void AnalyzeImage(State state, @Nullable AWTCanvas screenshot); + /** + * Analyze the captured image and update the verdict. + * @param state The state of the application. + * @param screenshot The captured screenshot of the current state. + * @param widget Optional, the corresponding widget when the screenshot is an action shot. + */ + void AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget); + /** * Destroy the visual validation manager. */ diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index e24113b9d..772135f3c 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -23,6 +23,7 @@ import org.fruit.alayer.State; import org.fruit.alayer.StrokePattern; import org.fruit.alayer.Verdict; +import org.fruit.alayer.Widget; import org.fruit.alayer.visualizers.TextVisualizer; import org.testar.Logger; @@ -68,6 +69,11 @@ public VisualValidator(@NonNull VisualValidationSettings settings) { @Override public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { + AnalyzeImage(state, screenshot, null); + } + + @Override + public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { // Create new session startNewAnalysis(); @@ -75,7 +81,7 @@ public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { parseScreenshot(screenshot); // Start extracting text, provide callback once finished. - extractExpectedText(state); + extractExpectedText(state, widget); // Match the expected text with the detected text. matchText(); @@ -106,8 +112,8 @@ private void parseScreenshot(@Nullable AWTCanvas screenshot) { } } - private void extractExpectedText(State state) { - _extractor.ExtractExpectedText(state, this); + private void extractExpectedText(State state, @Nullable Widget widget) { + _extractor.ExtractExpectedText(state, widget, this); } private void matchText() { diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java b/testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java index 698025967..9d7465c41 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/DummyExtractor.java @@ -1,13 +1,15 @@ package nl.ou.testar.visualvalidation.extractor; +import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.State; +import org.fruit.alayer.Widget; import java.util.ArrayList; public class DummyExtractor implements TextExtractorInterface{ @Override - public void ExtractExpectedText(State state, ExpectedTextCallback callback) { - callback.ReportExtractedText(new ArrayList<>()); + public void ExtractExpectedText(State state, @Nullable Widget widget, ExpectedTextCallback callback) { + } @Override diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java index 883c781c6..458004174 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -2,6 +2,7 @@ import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.Environment; import org.fruit.Util; import org.fruit.alayer.Roles; @@ -28,37 +29,31 @@ public class ExpectedTextExtractorBase extends Thread implements TextExtractorInterface { private static final String TAG = "ExpectedTextExtractor"; - - AtomicBoolean running = new AtomicBoolean(true); - + /** + * Lookup table to map the name of the name of a {@link Tag}, to the actual {@link Tag}. + * Holds all the available tag's which could hold text (String types, only). The key represents the {@link Tag} in String type, value + */ + @SuppressWarnings("unchecked") + private static final Map> _tag = TagsBase.tagSet().stream() + .filter(tag -> tag.type().equals(String.class)) + .collect(Collectors.toMap(Tag::name, tag -> (Tag) tag)); private final Boolean _threadSync = true; - private org.fruit.alayer.State _state = null; - private ExpectedTextCallback _callback = null; - final private Tag defaultTag; - private final boolean _loggingEnabled; - /** * A blacklist for {@link Widget}'s which should be ignored based on their {@code Role} because the don't contain * readable text. Optional when the value (which represents the ancestor path) is set, the {@link Widget} should * only be ignored when the ancestor path is equal with the {@link Widget} under investigation. */ private final Map> _blacklist = new HashMap<>(); - /** * A lookup table which indicates based on the {@code Role} which {@link Tag} should be used to extract the text. */ private final Map _lookupTable = new HashMap<>(); - - /** - * Lookup table to map the name of the name of a {@link Tag}, to the actual {@link Tag}. - * Holds all the available tag's which could hold text (String types, only). The key represents the {@link Tag} in String type, value - */ - @SuppressWarnings("unchecked") - private static final Map> _tag = TagsBase.tagSet().stream() - .filter(tag -> tag.type().equals(String.class)) - .collect(Collectors.toMap(Tag::name, tag -> (Tag) tag)); + AtomicBoolean running = new AtomicBoolean(true); + private org.fruit.alayer.State _state = null; + private ExpectedTextCallback _callback = null; + private Widget _widget = null; ExpectedTextExtractorBase(Tag defaultTag) { WidgetTextConfiguration config = ExtendedSettingsFactory.createWidgetTextConfiguration(); @@ -82,10 +77,22 @@ public class ExpectedTextExtractorBase extends Thread implements TextExtractorIn start(); } + static Rectangle getLocation(Widget widget) { + Shape dimension = widget.get(Shape, null); + + int x = dimension != null ? (int) dimension.x() : 0; + int y = dimension != null ? (int) dimension.y() : 0; + int width = dimension != null ? (int) dimension.width() : 0; + int height = dimension != null ? (int) dimension.height() : 0; + + return new Rectangle(x, y, width, height); + } + @Override - public void ExtractExpectedText(org.fruit.alayer.State state, ExpectedTextCallback callback) { + public void ExtractExpectedText(org.fruit.alayer.State state, @Nullable Widget widget, ExpectedTextCallback callback) { synchronized (_threadSync) { _state = state; + _widget = widget; _callback = callback; if (_loggingEnabled) { Logger.log(Level.TRACE, TAG, "Queue new text extract."); @@ -114,17 +121,6 @@ public void run() { } } - static Rectangle getLocation(Widget widget) { - Shape dimension = widget.get(Shape, null); - - int x = dimension != null ? (int) dimension.x() : 0; - int y = dimension != null ? (int) dimension.y() : 0; - int width = dimension != null ? (int) dimension.width() : 0; - int height = dimension != null ? (int) dimension.height() : 0; - - return new Rectangle(x, y, width, height); - } - private void extractText() { if (_state == null || _callback == null) { Logger.log(Level.ERROR, TAG, "Should not try to extract text on empty state/callback"); @@ -144,30 +140,37 @@ private void extractText() { double displayScale = Environment.getInstance().getDisplayScale(_state.get(Tags.HWND, (long) 0)); List expectedElements = new ArrayList<>(); - for (Widget widget : _state) { - String widgetRole = widget.get(Role).name(); - String text = widget.get(getVisualTextTag(widgetRole), ""); - - if (widgetIsIncluded(widget, widgetRole)) { - if (text != null && !text.isEmpty()) { - Rectangle absoluteLocation = getLocation(widget); - Rectangle relativeLocation = new Rectangle( - (int) ((absoluteLocation.x - applicationPosition.x) * displayScale), - (int) ((absoluteLocation.y - applicationPosition.y) * displayScale), - (int) (absoluteLocation.width * displayScale), - (int) (absoluteLocation.height * displayScale)); - expectedElements.add(new ExpectedElement(relativeLocation, text)); - } - } else { - if (_loggingEnabled) { - Logger.log(Level.DEBUG, TAG, "Widget {} with role {} is ignored", text, widgetRole); - } - } + if (_widget != null) { + extractedText(getLocation(_widget), displayScale, expectedElements, _widget); + } else { + Rectangle finalApplicationPosition = applicationPosition; + _state.forEach(widget -> extractedText(finalApplicationPosition, displayScale, expectedElements, widget)); } _callback.ReportExtractedText(expectedElements); } + private void extractedText(Rectangle applicationPosition, double displayScale, List expectedElements, Widget widget) { + String widgetRole = widget.get(Role).name(); + String text = widget.get(getVisualTextTag(widgetRole), ""); + + if (widgetIsIncluded(widget, widgetRole)) { + if (text != null && !text.isEmpty()) { + Rectangle absoluteLocation = getLocation(widget); + Rectangle relativeLocation = new Rectangle( + (int) ((absoluteLocation.x - applicationPosition.x) * displayScale), + (int) ((absoluteLocation.y - applicationPosition.y) * displayScale), + (int) (absoluteLocation.width * displayScale), + (int) (absoluteLocation.height * displayScale)); + expectedElements.add(new ExpectedElement(relativeLocation, text)); + } + } else { + if (_loggingEnabled) { + Logger.log(Level.DEBUG, TAG, "Widget {} with role {} is ignored", text, widgetRole); + } + } + } + protected boolean widgetIsIncluded(Widget widget, String role) { boolean containsReadableText = true; if (_blacklist.containsKey(role)) { diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java b/testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java index 120690650..cae12004a 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/TextExtractorInterface.java @@ -1,5 +1,6 @@ package nl.ou.testar.visualvalidation.extractor; +import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.State; import org.fruit.alayer.Widget; @@ -8,9 +9,10 @@ public interface TextExtractorInterface { * Extract the expected text for all the available {@link Widget}'s in the given {@link State}. * * @param state The current state of the application under test. + * @param widget When set we only extract the text from this widget instead of the entire state. * @param callback Callback function for returning the expected text. */ - void ExtractExpectedText(State state, ExpectedTextCallback callback); + void ExtractExpectedText(State state, @Nullable Widget widget, ExpectedTextCallback callback); /** * Destroy the text extractor. diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index 5f5ad5626..02e682319 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -38,6 +38,7 @@ import static org.fruit.alayer.Tags.ExecutedAction; import static org.fruit.alayer.Tags.IsRunning; import static org.fruit.alayer.Tags.OracleVerdict; +import static org.fruit.alayer.Tags.OriginWidget; import static org.fruit.alayer.Tags.SystemState; import static org.fruit.monkey.ConfigTags.LogLevel; import static org.fruit.monkey.ConfigTags.ProtocolClass; @@ -1735,13 +1736,16 @@ else if (actions.isEmpty()){ //TODO move the CPU metric to another helper class that is not default "TrashBinCode" or "SUTprofiler" //TODO check how well the CPU usage based waiting works protected boolean executeAction(SUT system, State state, Action action){ - + AWTCanvas screenshot = null; if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)){ - WdProtocolUtil.getActionshot(state,action); + screenshot = WdProtocolUtil.getActionshot(state,action); }else{ - ProtocolUtil.getActionshot(state,action); + screenshot = ProtocolUtil.getActionshot(state,action); } - + ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), screenshot); + + visualValidationManager.AnalyzeImage(state, screenshot, action.get(OriginWidget, null)); + double waitTime = settings.get(ConfigTags.TimeToWaitAfterAction); try{ diff --git a/webdriver/src/org/fruit/alayer/webdriver/WdProtocolUtil.java b/webdriver/src/org/fruit/alayer/webdriver/WdProtocolUtil.java index e477e77b2..a994686c5 100644 --- a/webdriver/src/org/fruit/alayer/webdriver/WdProtocolUtil.java +++ b/webdriver/src/org/fruit/alayer/webdriver/WdProtocolUtil.java @@ -54,7 +54,7 @@ public static String getStateshot(State state) { return ScreenshotSerialiser.saveStateshot(state.get(Tags.ConcreteIDCustom), screenshot); } - public static String getActionshot(State state, Action action) { + public static AWTCanvas getActionshot(State state, Action action) { List targets = action.get(Tags.Targets, null); if (targets == null) { return null; @@ -81,8 +81,7 @@ public static String getActionshot(State state, Action action) { Rect rect = Rect.from( actionArea.x, actionArea.y, actionArea.width + 1, actionArea.height + 1); - AWTCanvas scrshot = WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); - return ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), scrshot); + return WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); } public static AWTCanvas getStateshotBinary(State state) { @@ -91,9 +90,8 @@ public static AWTCanvas getStateshotBinary(State state) { && state.get(WdTags.WebHorizontallyScrollable, null) == null) { //Get a screenshot of all the screen, because SUT ended and we can't obtain the size Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); - AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(screenRect.getX(), screenRect.getY(), + return AWTCanvas.fromScreenshot(Rect.from(screenRect.getX(), screenRect.getY(), screenRect.getWidth(), screenRect.getHeight()), state.get(Tags.HWND, (long)0), AWTCanvas.StorageFormat.PNG, 1); - return scrshot; } double width = CanvasDimensions.getCanvasWidth() + ( @@ -101,7 +99,6 @@ public static AWTCanvas getStateshotBinary(State state) { double height = CanvasDimensions.getCanvasHeight() + ( state.get(WdTags.WebHorizontallyScrollable) ? scrollThick : 0); Rect rect = Rect.from(0, 0, width, height); - AWTCanvas screenshot = WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); - return screenshot; + return WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); } -} \ No newline at end of file +} From b7108a964c18ecb341b10181492d454d9d5f9b4b Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 2 Jul 2021 22:15:58 +0200 Subject: [PATCH 26/40] Fixed flaky test and added suppression for spamming log messages. --- .../testar/visualvalidation/TextElement.java | 11 ++++- .../matcher/ContentMatchResult.java | 27 ++++++++---- .../matcher/ContentMatcher.java | 42 ++++++++++++++----- .../matcher/LocationMatch.java | 14 +++++-- .../matcher/LocationMatcher.java | 14 ++++--- .../matcher/MatcherConfiguration.java | 4 +- .../matcher/MatcherResult.java | 8 ++-- .../matcher/ContentMatcherTest.java | 10 +++-- 8 files changed, 93 insertions(+), 37 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/TextElement.java b/testar/src/nl/ou/testar/visualvalidation/TextElement.java index c007ab1d6..0770d0f06 100644 --- a/testar/src/nl/ou/testar/visualvalidation/TextElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/TextElement.java @@ -2,7 +2,7 @@ import java.awt.Rectangle; -public class TextElement { +public class TextElement implements Comparable { public final Rectangle _location; public final String _text; @@ -24,4 +24,13 @@ public String toString() { ", _text='" + _text + '\'' + '}'; } + + @Override + public int compareTo(TextElement other) { + int result = -1; + if (_text.equals(other._text) && _location.equals(other._location)) { + result = 0; + } + return result; + } } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java index 8073c7083..caa2ef792 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java @@ -7,9 +7,10 @@ /** * Holds the matching result for the content of the expected text. */ -public class ContentMatchResult { +public class ContentMatchResult implements Comparable { public final ExpectedTextMatchResult expectedResult; public final RecognizedTextMatchResult recognizedResult; + public final String expectedText; public final long totalMatched; public final long totalExpected; @@ -20,6 +21,12 @@ public ContentMatchResult(ExpectedTextMatchResult expectedResult, RecognizedText .filter(e -> e.result != CharacterMatchResult.NO_MATCH) .count(); totalExpected = expectedResult.expectedText.size(); + expectedText = expectedResult.expectedText.stream(). + map(e -> e.character.character).collect(Collector.of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString)); } @Override @@ -27,12 +34,7 @@ public String toString() { StringBuilder str = new StringBuilder() .append("\n") - .append("Matched \"").append(expectedResult.expectedText.stream(). - map(e -> e.character.character).collect(Collector.of( - StringBuilder::new, - StringBuilder::append, - StringBuilder::append, - StringBuilder::toString))) + .append("Matched \"").append(expectedText) .append("\"\n") .append("Result: ") @@ -80,4 +82,15 @@ public String toString() { return str.toString(); } + + @Override + public int compareTo(ContentMatchResult other) { + int result = -1; + if (expectedText.equals(other.expectedText) && + (totalExpected == other.totalExpected) && + (totalMatched == other.totalMatched)) { + result = 0; + } + return result; + } } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java index d5dda3ffa..2698b0935 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java @@ -24,15 +24,22 @@ class ContentMatcher { * mechanism is a grid from left to right and top to bottom. The algorithm tries to find a matching character from * the grid, once found the index is stored and will be used as the starting position for the next iteration. */ - public static ContentMatchResult Match(LocationMatch locationMatch) { + public static ContentMatchResult Match(LocationMatch locationMatch, MatcherConfiguration configuration) { String expectedText = locationMatch.expectedElement._text; ExpectedTextMatchResult expectedResult = new ExpectedTextMatchResult(expectedText); - Logger.log(Level.INFO, TAG, "Expected text @{} : {}", locationMatch.expectedElement._location, expectedResult); + if (configuration.loggingEnabled) { + Logger.log(Level.INFO, TAG, + "Expected text @{} : {}", + locationMatch.expectedElement._location, + expectedResult); + } // Sort the recognized elements. - List sorted = sortRecognizedElements(locationMatch); + List sorted = sortRecognizedElements(locationMatch, configuration); RecognizedTextMatchResult recognizedResult = new RecognizedTextMatchResult(sorted); - Logger.log(Level.INFO, TAG, "Recognized text: {}", recognizedResult); + if (configuration.loggingEnabled) { + Logger.log(Level.INFO, TAG, "Recognized text: {}", recognizedResult); + } // Iterate over the expected text and try to match with a recognized element character. int unMatchedSize = recognizedResult.recognized.size(); @@ -73,8 +80,10 @@ public static ContentMatchResult Match(LocationMatch locationMatch) { } @NonNull - static List sortRecognizedElements(LocationMatch locationMatch) { - Map> lineBucket = sortRecognizedElementsPerTextLine(locationMatch); + static List sortRecognizedElements(LocationMatch locationMatch, + MatcherConfiguration configuration) { + Map> lineBucket = + sortRecognizedElementsPerTextLine(locationMatch, configuration); // Get the sorted Y-axe coordinates for the identified lines. List bucketSorted = lineBucket.keySet().stream().sorted().collect(Collectors.toList()); @@ -93,7 +102,8 @@ static List sortRecognizedElements(LocationMatch locationMatc } @NonNull - static Map> sortRecognizedElementsPerTextLine(LocationMatch locationMatch) { + static Map> sortRecognizedElementsPerTextLine(LocationMatch locationMatch, + MatcherConfiguration configuration) { // Determine the average height of the recognized text elements. double lineHeight = locationMatch.recognizedElements.stream() .mapToInt(e -> e._location.height) @@ -115,16 +125,26 @@ static Map> sortRecognizedElementsPerTextLine(L if (IntStream.rangeClosed(c - margin, c + margin) .boxed() .collect(Collectors.toList()) - .contains(line.left())) - { - Logger.log(Level.DEBUG, TAG, "Line {} {} in range of bucket {}", line.left(), line.right(), c); + .contains(line.left())) { + if (configuration.loggingEnabled) { + Logger.log(Level.DEBUG, TAG, + "Line {} {} in range of bucket {}", + line.left(), + line.right(), + c); + } lineBucket.get(c).add(line.right()); foundBucket = true; break; } } if (!foundBucket) { - Logger.log(Level.DEBUG, TAG, "No bucket found creating new one for {} {}", line.left(), line.right()); + if (configuration.loggingEnabled) { + Logger.log(Level.DEBUG, TAG, + "No bucket found creating new one for {} {}", + line.left(), + line.right()); + } lineBucket.put(line.left(), new ArrayList<>()); lineBucket.get(line.left()).add(line.right()); } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatch.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatch.java index f8f565646..e1232af42 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatch.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatch.java @@ -3,14 +3,14 @@ import nl.ou.testar.visualvalidation.extractor.ExpectedElement; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; -import java.util.HashSet; import java.util.Set; +import java.util.TreeSet; -public class LocationMatch { +public class LocationMatch implements Comparable { final public MatchLocation location; final public ExpectedElement expectedElement; - public Set recognizedElements = new HashSet<>(); + public Set recognizedElements = new TreeSet<>(); public LocationMatch(ExpectedElement expectedElement, int margin) { this.location = new MatchLocation(margin, expectedElement._location); @@ -29,4 +29,12 @@ public String toString() { ", recognizedElements=" + recognizedElements + '}'; } + + @Override + public int compareTo(LocationMatch other) { + int result = -1; + if (location.equals(other.location) && expectedElement.equals(other.expectedElement)) { + result = 0; + } + return result; } } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java index 232a51dc8..89ecb9258 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java @@ -9,9 +9,9 @@ import java.awt.Dimension; import java.awt.Rectangle; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; public class LocationMatcher implements VisualMatcher { @@ -58,7 +58,7 @@ public MatcherResult Match(List recognizedElements, List recognizedElements, List recognizedElements) { - Set removeRecognizedList = new HashSet<>(); + Set removeRecognizedList = new TreeSet<>(); matcherResult.getLocationMatches().forEach(it -> removeRecognizedList.addAll(it.recognizedElements)); // Remove the elements which have been matched with expected elements. @@ -86,9 +86,11 @@ void setUnmatchedElements(MatcherResult matcherResult, List r // The remainder of the recognized elements can be considered to be unmatched. recognizedElements.forEach(matcherResult::addNoLocationMatch); - Logger.log(Level.INFO, TAG, "No location match for the following detected text elements:\n {}", matcherResult.getNoLocationMatches().stream() - .map(e -> e._text + " " + e._location + "\n") - .collect(Collectors.joining())); + if (config.loggingEnabled) { + Logger.log(Level.INFO, TAG, "No location match for the following detected text elements:\n {}", matcherResult.getNoLocationMatches().stream() + .map(e -> e._text + " " + e._location + "\n") + .collect(Collectors.joining())); + } } /** diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java index a5079119f..b34960a16 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java @@ -8,6 +8,7 @@ @XmlAccessorType(XmlAccessType.FIELD) public class MatcherConfiguration extends ExtendedSettingBase { public Integer locationMatchMargin; + boolean loggingEnabled; @Override public String toString() { @@ -19,13 +20,14 @@ public String toString() { public static MatcherConfiguration CreateDefault() { MatcherConfiguration instance = new MatcherConfiguration(); instance.locationMatchMargin = 0; + instance.loggingEnabled = false; return instance; } @Override public int compareTo(MatcherConfiguration other) { int result = -1; - if (locationMatchMargin.equals(other.locationMatchMargin)) { + if (locationMatchMargin.equals(other.locationMatchMargin) && (loggingEnabled == other.loggingEnabled)) { result = 0; } return result; diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java index bee71b376..8fd682a0a 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java @@ -2,13 +2,13 @@ import nl.ou.testar.visualvalidation.TextElement; -import java.util.HashSet; import java.util.Set; +import java.util.TreeSet; public class MatcherResult { - private final Set noMatches = new HashSet<>(); - private final Set locationMatches = new HashSet<>(); - private final Set contentMatchResults = new HashSet<>(); + private final Set noMatches = new TreeSet<>(); + private final Set locationMatches = new TreeSet<>(); + private final Set contentMatchResults = new TreeSet<>(); public void addContentMatchResult(ContentMatchResult result) { contentMatchResults.add(result); diff --git a/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java b/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java index 30068e7c7..b58abcd25 100644 --- a/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java +++ b/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java @@ -23,6 +23,8 @@ public class ContentMatcherTest { private static final RecognizedElement thirdLineSecond = new RecognizedElement(new Rectangle(10, 25, 20, 5), 100, "?"); private static final ExpectedElement expectedElement = new ExpectedElement(new Rectangle(0, 0, 20, 30), "1K\n\r2w\n\r3"); + private static final MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + private LocationMatch prepareExpectedTextWith3Lines() { LocationMatch locationMatch = new LocationMatch(expectedElement, 0); // Shuffled input so we can test the algorithm. @@ -40,7 +42,7 @@ public void match() { LocationMatch input = prepareExpectedTextWith3Lines(); // WHEN we run the algorithm. - ContentMatchResult result = ContentMatcher.Match(input); + ContentMatchResult result = ContentMatcher.Match(input, config); // THEN the result must be: assertEquals(9, result.totalExpected); @@ -92,7 +94,7 @@ public void sortRecognizedElements() { LocationMatch input = prepareExpectedTextWith3Lines(); // WHEN we sort the recognized elements. - List result = ContentMatcher.sortRecognizedElements(input); + List result = ContentMatcher.sortRecognizedElements(input, config); // THEN the recognized elements should be sorted correctly. assertEquals(Arrays.asList(firstLine, secondLine, thirdLineFirst, thirdLineSecond), result); @@ -104,13 +106,13 @@ public void sortRecognizedElementsPerTextLine() { LocationMatch input = prepareExpectedTextWith3Lines(); // WHEN we sort the recognized elements only on their y coordinate. - Map> result = ContentMatcher.sortRecognizedElementsPerTextLine(input); + Map> result = ContentMatcher.sortRecognizedElementsPerTextLine(input, config); // THEN the recognized elements should be sorted correctly. Map> expectedResult = Stream.of( new AbstractMap.SimpleEntry<>(5, Collections.singletonList(firstLine)), new AbstractMap.SimpleEntry<>(15, Collections.singletonList(secondLine)), - new AbstractMap.SimpleEntry<>(25, Arrays.asList(thirdLineFirst, thirdLineSecond))) + new AbstractMap.SimpleEntry<>(27, Arrays.asList(thirdLineFirst, thirdLineSecond))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); expectedResult.keySet().forEach(expectedKey -> { From 43eac807de3b1f69974d212a19e9bbaa902ff453 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sun, 25 Jul 2021 16:47:37 +0200 Subject: [PATCH 27/40] Introduced Location as more print friendly alternative of Rectangle. --- .../ou/testar/visualvalidation/Location.java | 21 +++++++++++++++++++ .../testar/visualvalidation/TextElement.java | 4 ++-- .../extractor/ExpectedElement.java | 5 ++--- .../extractor/ExpectedTextExtractorBase.java | 3 ++- .../matcher/ContentMatchResult.java | 6 +++++- .../matcher/MatchLocation.java | 6 +++--- .../ocr/RecognizedElement.java | 3 ++- .../ocr/tesseract/TesseractResult.java | 8 +++---- .../matcher/ContentMatcherTest.java | 12 +++++------ .../matcher/LocationMatcherTest.java | 12 +++++------ 10 files changed, 53 insertions(+), 27 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/Location.java diff --git a/testar/src/nl/ou/testar/visualvalidation/Location.java b/testar/src/nl/ou/testar/visualvalidation/Location.java new file mode 100644 index 000000000..842ead4ff --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/Location.java @@ -0,0 +1,21 @@ +package nl.ou.testar.visualvalidation; + +import java.awt.Rectangle; + +public class Location extends Rectangle { + public Location(int x, int y, int width, int height) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + + public Location() { + + } + + @Override + public String toString() { + return String.format("[x=%d, y=%d, width=%d, height=%d]", x, y, width, height); + } +} diff --git a/testar/src/nl/ou/testar/visualvalidation/TextElement.java b/testar/src/nl/ou/testar/visualvalidation/TextElement.java index 0770d0f06..0b66a16ed 100644 --- a/testar/src/nl/ou/testar/visualvalidation/TextElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/TextElement.java @@ -3,7 +3,7 @@ import java.awt.Rectangle; public class TextElement implements Comparable { - public final Rectangle _location; + public final Location _location; public final String _text; /** @@ -12,7 +12,7 @@ public class TextElement implements Comparable { * @param location The relative location of the text inside the application. * @param text The text. */ - public TextElement(Rectangle location, String text) { + public TextElement(Location location, String text) { _location = location; _text = text; } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java index 2d3b31795..624f657cc 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedElement.java @@ -1,9 +1,8 @@ package nl.ou.testar.visualvalidation.extractor; +import nl.ou.testar.visualvalidation.Location; import nl.ou.testar.visualvalidation.TextElement; -import java.awt.Rectangle; - public class ExpectedElement extends TextElement { /** @@ -12,7 +11,7 @@ public class ExpectedElement extends TextElement { * @param location The relative location of the text inside the application. * @param text The text. */ - public ExpectedElement(Rectangle location, String text) { + public ExpectedElement(Location location, String text) { super(location, text); } } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java index 458004174..022ff593c 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -1,6 +1,7 @@ package nl.ou.testar.visualvalidation.extractor; +import nl.ou.testar.visualvalidation.Location; import org.apache.logging.log4j.Level; import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.Environment; @@ -157,7 +158,7 @@ private void extractedText(Rectangle applicationPosition, double displayScale, L if (widgetIsIncluded(widget, widgetRole)) { if (text != null && !text.isEmpty()) { Rectangle absoluteLocation = getLocation(widget); - Rectangle relativeLocation = new Rectangle( + Location relativeLocation = new Location( (int) ((absoluteLocation.x - applicationPosition.x) * displayScale), (int) ((absoluteLocation.y - applicationPosition.y) * displayScale), (int) (absoluteLocation.width * displayScale), diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java index caa2ef792..276cd7684 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java @@ -1,5 +1,7 @@ package nl.ou.testar.visualvalidation.matcher; +import nl.ou.testar.visualvalidation.Location; + import java.util.List; import java.util.stream.Collector; import java.util.stream.Collectors; @@ -11,12 +13,14 @@ public class ContentMatchResult implements Comparable { public final ExpectedTextMatchResult expectedResult; public final RecognizedTextMatchResult recognizedResult; public final String expectedText; + public final Location foundLocation; public final long totalMatched; public final long totalExpected; - public ContentMatchResult(ExpectedTextMatchResult expectedResult, RecognizedTextMatchResult recognizedResult) { + public ContentMatchResult(ExpectedTextMatchResult expectedResult, RecognizedTextMatchResult recognizedResult, Location foundLocation) { this.expectedResult = expectedResult; this.recognizedResult = recognizedResult; + this.foundLocation = foundLocation; totalMatched = expectedResult.expectedText.stream() .filter(e -> e.result != CharacterMatchResult.NO_MATCH) .count(); diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java index 3217ad10b..8063187a7 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatchLocation.java @@ -1,12 +1,12 @@ package nl.ou.testar.visualvalidation.matcher; -import java.awt.Rectangle; +import nl.ou.testar.visualvalidation.Location; public class MatchLocation { final public int margin; - final public Rectangle location; + final public Location location; - public MatchLocation(int margin, Rectangle location) { + public MatchLocation(int margin, Location location) { this.margin = margin; this.location = location; } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java index 03b208e52..6ad308502 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/RecognizedElement.java @@ -1,5 +1,6 @@ package nl.ou.testar.visualvalidation.ocr; +import nl.ou.testar.visualvalidation.Location; import nl.ou.testar.visualvalidation.TextElement; import java.awt.Rectangle; @@ -17,7 +18,7 @@ public class RecognizedElement extends TextElement { * @param confidence The confidence level of the discovered text. * @param text The discovered text. */ - public RecognizedElement(Rectangle location, float confidence, String text) { + public RecognizedElement(Location location, float confidence, String text) { super(location, text); _confidence = confidence; } diff --git a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java index ced39ce7a..b718434e8 100644 --- a/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/ocr/tesseract/TesseractResult.java @@ -1,5 +1,6 @@ package nl.ou.testar.visualvalidation.ocr.tesseract; +import nl.ou.testar.visualvalidation.Location; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.apache.logging.log4j.Level; import org.bytedeco.javacpp.BytePointer; @@ -7,7 +8,6 @@ import org.bytedeco.tesseract.ResultIterator; import org.testar.Logger; -import java.awt.Rectangle; import java.util.function.Supplier; /** @@ -27,9 +27,9 @@ static RecognizedElement Extract(ResultIterator recognizedElement, int granulari Supplier intPointerSupplier = () -> new IntPointer(new int[1]); BytePointer ocrResult = recognizedElement.GetUTF8Text(granularity); - if (ocrResult == null){ + if (ocrResult == null) { Logger.log(Level.ERROR, "OCR", "Results is null"); - return new RecognizedElement(new Rectangle(), 0, ""); + return new RecognizedElement(new Location(), 0, ""); } String recognizedText = ocrResult.getString().trim(); @@ -47,7 +47,7 @@ static RecognizedElement Extract(ResultIterator recognizedElement, int granulari // Upper left coordinate = 0,0 int width = right.get() - left.get(); int height = bottom.get() - top.get(); - Rectangle location = new Rectangle(left.get(), top.get(), width, height); + Location location = new Location(left.get(), top.get(), width, height); RecognizedElement result = new RecognizedElement(location, confidence, recognizedText); left.deallocate(); diff --git a/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java b/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java index b58abcd25..f34e5f42d 100644 --- a/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java +++ b/testar/test/nl/ou/testar/visualvalidation/matcher/ContentMatcherTest.java @@ -1,10 +1,10 @@ package nl.ou.testar.visualvalidation.matcher; +import nl.ou.testar.visualvalidation.Location; import nl.ou.testar.visualvalidation.extractor.ExpectedElement; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.junit.Test; -import java.awt.Rectangle; import java.util.AbstractMap; import java.util.Arrays; import java.util.Collections; @@ -17,11 +17,11 @@ public class ContentMatcherTest { - private static final RecognizedElement firstLine = new RecognizedElement(new Rectangle(0, 0, 20, 10), 100, "1"); - private static final RecognizedElement secondLine = new RecognizedElement(new Rectangle(0, 10, 20, 10), 100, "2W"); - private static final RecognizedElement thirdLineFirst = new RecognizedElement(new Rectangle(0, 20, 20, 10), 100, "3 "); - private static final RecognizedElement thirdLineSecond = new RecognizedElement(new Rectangle(10, 25, 20, 5), 100, "?"); - private static final ExpectedElement expectedElement = new ExpectedElement(new Rectangle(0, 0, 20, 30), "1K\n\r2w\n\r3"); + private static final RecognizedElement firstLine = new RecognizedElement(new Location(0, 0, 20, 10), 100, "1"); + private static final RecognizedElement secondLine = new RecognizedElement(new Location(0, 10, 20, 10), 100, "2W"); + private static final RecognizedElement thirdLineFirst = new RecognizedElement(new Location(0, 20, 20, 10), 100, "3 "); + private static final RecognizedElement thirdLineSecond = new RecognizedElement(new Location(10, 25, 20, 5), 100, "?"); + private static final ExpectedElement expectedElement = new ExpectedElement(new Location(0, 0, 20, 30), "1K\n\r2w\n\r3"); private static final MatcherConfiguration config = MatcherConfiguration.CreateDefault(); diff --git a/testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java b/testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java index c165addde..f11ff74d2 100644 --- a/testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java +++ b/testar/test/nl/ou/testar/visualvalidation/matcher/LocationMatcherTest.java @@ -1,5 +1,6 @@ package nl.ou.testar.visualvalidation.matcher; +import nl.ou.testar.visualvalidation.Location; import nl.ou.testar.visualvalidation.extractor.ExpectedElement; import nl.ou.testar.visualvalidation.ocr.RecognizedElement; import org.junit.Test; @@ -8,7 +9,6 @@ import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; -import java.awt.Rectangle; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -23,15 +23,15 @@ public class LocationMatcherTest { private static final RecognizedElement recognizedElementOne = - new RecognizedElement(new Rectangle(0, 15, 36, 1), 0, "r"); + new RecognizedElement(new Location(0, 15, 36, 1), 0, "r"); private static final RecognizedElement recognizedElementTwo = - new RecognizedElement(new Rectangle(40, 16, 26, 45), 23, "df"); + new RecognizedElement(new Location(40, 16, 26, 45), 23, "df"); private static final RecognizedElement recognizedElementThree = - new RecognizedElement(new Rectangle(5, 10, 26, 45), 23, "aa"); + new RecognizedElement(new Location(5, 10, 26, 45), 23, "aa"); private static final RecognizedElement recognizedElementFour = - new RecognizedElement(new Rectangle(12, 16, 10, 20), 23, "Open"); + new RecognizedElement(new Location(12, 16, 10, 20), 23, "Open"); private static final ExpectedElement expectedElement = - new ExpectedElement(new Rectangle(10, 15, 15, 23), "open"); + new ExpectedElement(new Location(10, 15, 15, 23), "open"); @Test public void successfulMatch() { From fef990d61dc7dc39e03523544cf9306d157079e5 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 28 Jul 2021 15:18:00 +0200 Subject: [PATCH 28/40] Added verdict support and extended HTML report with validation result. --- .../serialisation/ScreenshotSerialiser.java | 2 + core/src/org/fruit/alayer/AWTCanvas.java | 63 ++- .../HtmlReporting/HtmlSequenceReport.java | 450 +++++++++++------- .../nl/ou/testar/HtmlReporting/Reporting.java | 3 + .../HtmlReporting/XMLSequenceReport.java | 7 + .../DummyVisualValidator.java | 9 +- .../VisualValidationManager.java | 7 +- .../visualvalidation/VisualValidationTag.java | 9 + .../VisualValidationVerdict.java | 31 ++ .../visualvalidation/VisualValidator.java | 142 ++++-- .../matcher/CharacterMatch.java | 14 + .../matcher/CharacterMatchEntry.java | 16 +- .../matcher/CharacterMatchResult.java | 20 +- .../matcher/ContentMatchResult.java | 15 +- .../matcher/ContentMatcher.java | 10 +- .../matcher/ExpectedTextMatchResult.java | 10 +- .../matcher/MatcherResult.java | 6 + .../matcher/RecognizedTextMatchResult.java | 7 + .../src/org/fruit/monkey/DefaultProtocol.java | 56 ++- .../org/testar/protocols/DesktopProtocol.java | 1 - .../testar/protocols/WebdriverProtocol.java | 1 - 21 files changed, 586 insertions(+), 293 deletions(-) create mode 100644 testar/src/nl/ou/testar/visualvalidation/VisualValidationTag.java create mode 100644 testar/src/nl/ou/testar/visualvalidation/VisualValidationVerdict.java diff --git a/core/src/es/upv/staq/testar/serialisation/ScreenshotSerialiser.java b/core/src/es/upv/staq/testar/serialisation/ScreenshotSerialiser.java index 07fbc2963..3e434b7a6 100644 --- a/core/src/es/upv/staq/testar/serialisation/ScreenshotSerialiser.java +++ b/core/src/es/upv/staq/testar/serialisation/ScreenshotSerialiser.java @@ -109,6 +109,8 @@ public void run(){ r.scrshot.saveAsPng(r.scrshotPath); } catch (IOException e) { LogSerialiser.log("I/O exception saving screenshot <" + r.scrshotPath + ">\n", LogSerialiser.LogLevel.Critical); + } catch (NullPointerException e){ + LogSerialiser.log("Screenshot was empty" + r.scrshotPath + ">\n", LogSerialiser.LogLevel.Critical); } } } diff --git a/core/src/org/fruit/alayer/AWTCanvas.java b/core/src/org/fruit/alayer/AWTCanvas.java index 3d0600d8f..28dcadf1f 100644 --- a/core/src/org/fruit/alayer/AWTCanvas.java +++ b/core/src/org/fruit/alayer/AWTCanvas.java @@ -45,11 +45,12 @@ import java.awt.geom.AffineTransform; import java.awt.geom.Rectangle2D; import java.awt.image.BufferedImage; +import java.awt.image.ColorModel; import java.awt.image.DataBuffer; import java.awt.image.DataBufferInt; +import java.awt.image.WritableRaster; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; -import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -74,7 +75,7 @@ public class AWTCanvas implements Image, Canvas { - public static enum StorageFormat{ JPEG, PNG, BMP; } + public enum StorageFormat{ JPEG, PNG, BMP } public static void saveAsJpeg(BufferedImage image, OutputStream os, double quality) throws IOException{ if(quality == 1){ @@ -131,12 +132,8 @@ public static AWTCanvas fromScreenshot(Rect r, long windowHandle, StorageFormat } public static AWTCanvas fromFile(String file) throws IOException{ - BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(file))); - - try{ + try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) { return fromInputStream(bis); - }finally{ - bis.close(); } } @@ -154,9 +151,9 @@ public static AWTCanvas fromInputStream(InputStream is) throws IOException{ private static final long serialVersionUID = -5041497503329308870L; protected transient BufferedImage img; - private StorageFormat format; - private double quality; - private double x, y; + private final StorageFormat format; + private final double quality; + private final double x, y; transient Graphics2D gr; static final Pen defaultPen = Pen.PEN_DEFAULT; double fontSize, strokeWidth; @@ -188,11 +185,8 @@ public AWTCanvas(double x, double y, BufferedImage image, StorageFormat format, this.format = format; this.quality = quality; gr = img.createGraphics(); - // gr.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, - // RenderingHints.VALUE_TEXT_ANTIALIAS_ON); - - adjustPen(defaultPen); - //gr.setComposite(AlphaComposite.Clear); + + adjustPen(defaultPen); } public void begin() {} @@ -203,7 +197,16 @@ public void end() {} public double x(){ return x; } public double y(){ return y; } public BufferedImage image(){ return img; } - + + /** + * @return A deep copy of the image. + */ + public BufferedImage deepCopyImage() { + ColorModel cm = img.getColorModel(); + WritableRaster raster = img.copyData(img.getRaster().createCompatibleWritableRaster()); + return new BufferedImage(cm, raster, cm.isAlphaPremultiplied(), null); + } + private void adjustPen(Pen pen){ Double tstrokeWidth = pen.strokeWidth(); if(tstrokeWidth == null) @@ -221,7 +224,7 @@ private void adjustPen(Pen pen){ strokePattern = tstrokePattern; strokeWidth = tstrokeWidth; strokeCaps = tstrokeCaps; - gr.setStroke(new BasicStroke((float)(double)strokeWidth)); + gr.setStroke(new BasicStroke((float)strokeWidth)); } Color tcolor = pen.color(); @@ -244,7 +247,7 @@ private void adjustPen(Pen pen){ if(!tfont.equals(font) || !tfontSize.equals(fontSize)){ font = tfont; fontSize = tfontSize; - gr.setFont(new Font(font, Font.PLAIN, (int)(double)fontSize)); + gr.setFont(new Font(font, Font.PLAIN, (int)fontSize)); } FillPattern tfillPattern = pen.fillPattern(); @@ -328,12 +331,8 @@ public void saveAsJpeg(OutputStream os, double quality) throws IOException{ } public void saveAsJpeg(String file, double quality) throws IOException{ - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(file))); - - try{ + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) { saveAsJpeg(bos, quality); - }finally{ - bos.close(); } } @@ -341,13 +340,9 @@ public void saveAsPng(OutputStream os) throws IOException{ saveAsPng(img, os); } - public void saveAsPng(String file) throws IOException{ - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(file))); - - try{ + public void saveAsPng(String file) throws IOException{ + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) { saveAsPng(bos); - }finally{ - bos.close(); } } @@ -375,7 +370,7 @@ public void paint(Canvas canvas, double x, double y, double width, double height) { Assert.notNull(canvas); - int data[] = ((DataBufferInt)img.getRaster().getDataBuffer()).getData(); + int[] data = ((DataBufferInt)img.getRaster().getDataBuffer()).getData(); canvas.image(canvas.defaultPen(), x, y, width, height, data, img.getWidth(), img.getHeight()); } @@ -392,7 +387,7 @@ public void paint(Canvas canvas, Rect srcRect, Rect destRect) { BufferedImage subImage = new BufferedImage(srcWidth, srcHeight, BufferedImage.TYPE_INT_ARGB); subImage.getGraphics().drawImage(img.getSubimage(srcX, srcY, srcWidth, srcHeight), 0, 0, srcWidth, srcHeight, null); - int area[] = ((DataBufferInt)subImage.getRaster().getDataBuffer()).getData(); + int[] area = ((DataBufferInt)subImage.getRaster().getDataBuffer()).getData(); canvas.image(canvas.defaultPen(), destRect.x(), destRect.y(), destRect.width(), destRect.height(), area, srcWidth, srcHeight); } @@ -412,7 +407,7 @@ public void triangle(Pen pen, double x1, double y1, double x2, double y2, * @author urueda */ public float compareImage(AWTCanvas img) { - //long now = System.currentTimeMillis(); + //long now = System.currentTimeMillis(); DataBuffer dbThis = this.img.getData().getDataBuffer(), dbImg = img.img.getData().getDataBuffer(); int sizeThis = dbThis.getSize(), @@ -435,10 +430,10 @@ else if (sizeThis < sizeImg) float meanSize = (sizeThis + sizeImg) / 2; float percent = sizeSimilarity - (1.0f - (equalPixels / meanSize)); //System.out.println("Image comparison took : " + (System.currentTimeMillis() - now) + " ms"); - return (percent < 0f ? 0f : (percent > 1f ? 1f : percent)); + return (percent < 0f ? 0f : Math.min(percent, 1f)); } public void release() {} - + public String toString(){ return "AWTCanvas (width: " + width() + " height: " + height() + ")"; } } diff --git a/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java b/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java index fa7c71baa..5657f21f5 100644 --- a/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java +++ b/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java @@ -32,7 +32,12 @@ package nl.ou.testar.HtmlReporting; import nl.ou.testar.a11y.reporting.HTMLReporter; +import nl.ou.testar.visualvalidation.matcher.CharacterMatchEntry; +import nl.ou.testar.visualvalidation.matcher.ContentMatchResult; +import nl.ou.testar.visualvalidation.matcher.LocationMatch; +import nl.ou.testar.visualvalidation.matcher.MatcherResult; import org.apache.commons.lang.StringEscapeUtils; +import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.Action; import org.fruit.alayer.State; import org.fruit.alayer.Tags; @@ -41,263 +46,388 @@ import java.io.File; import java.io.PrintWriter; +import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; -public class HtmlSequenceReport implements Reporting{ +public class HtmlSequenceReport implements Reporting { - private boolean firstStateAdded = false; - private boolean firstActionsAdded = false; - - private static final String[] HEADER = new String[] { + private static final String[] HEADER = new String[]{ "", "", + "", "", "TESTAR execution sequence report", "", "" }; - - private PrintWriter out; - private static final String REPORT_FILENAME_MID ="_sequence_"; + private static final String REPORT_FILENAME_MID = "_sequence_"; private static final String REPORT_FILENAME_AFT = ".html"; - + private boolean firstStateAdded = false; + private boolean firstActionsAdded = false; + private PrintWriter out; private int innerLoopCounter = 0; public HtmlSequenceReport() { - try{ + try { //TODO put filename into settings, name with sequence number // creating a new file for the report - String filename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString+"_" + String filename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString + "_" + OutputStructure.executedSUTname + REPORT_FILENAME_MID + OutputStructure.sequenceInnerLoopCount + REPORT_FILENAME_AFT; out = new PrintWriter(filename, HTMLReporter.CHARSET); - for(String s:HEADER){ + for (String s : HEADER) { write(s); } write("

TESTAR execution sequence report for sequence " + OutputStructure.sequenceInnerLoopCount + "

"); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } /** - * Constructor for Replay mode. - * - * @param mode + * Constructor for Replay mode. + * * @param pathReplayedSequence */ public HtmlSequenceReport(String pathReplayedSequence) { try { - String filename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString+"_" + String filename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString + "_" + OutputStructure.executedSUTname + REPORT_FILENAME_MID + OutputStructure.sequenceInnerLoopCount + REPORT_FILENAME_AFT; out = new PrintWriter(filename, HTMLReporter.CHARSET); - for(String s:HEADER) { + for (String s : HEADER) { write(s); } write("

TESTAR replay sequence report for file " + pathReplayedSequence + "

"); - } catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } - public void addTitle(int h, String text){ - write(""+text+""); + private static String correctScreenshotPath(String screenshotDir) { + if (screenshotDir.contains("./output")) { + int indexStart = screenshotDir.indexOf("./output"); + int indexScrn = screenshotDir.indexOf("scrshots"); + String replaceString = screenshotDir.substring(indexStart, indexScrn); + screenshotDir = screenshotDir.replace(replaceString, "../"); + } + return screenshotDir; + } + + static private String getScreenshotPath(State state) { + return correctScreenshotPath(state.get(Tags.ScreenshotPath)); + } + + static private String getActionScreenshotPath(State state, Action action) { + final String screenshotDir = correctScreenshotPath(OutputStructure.screenshotsOutputDir); + String actionPath = screenshotDir + File.separator + OutputStructure.startInnerLoopDateString + "_" + + OutputStructure.executedSUTname + "_sequence_" + OutputStructure.sequenceInnerLoopCount + + File.separator + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + "_" + + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + ".png"; + + if (actionPath.contains("./output")) { + actionPath = actionPath.replace("./output", ".."); + } + return actionPath; + } + + public void addTitle(int h, String text) { + write("" + text + ""); } - public void addSequenceStep(State state, String actionImagePath){ - try { - String imagePath = state.get(Tags.ScreenshotPath); - // repairing the file paths: - if(imagePath.contains("./output")){ - imagePath = imagePath.replace("./output","../"); - } - write("

State:

"); - write("

"); - write("

Action:

"); - write("

"); - }catch(Exception e) { - System.out.println("ERROR: Adding the Sequence step " + innerLoopCounter + " in the HTML report"); - write("

ERROR Adding current Sequence step " + innerLoopCounter + "

"); - } + public void addSequenceStep(State state, String actionImagePath) { + try { + String imagePath = state.get(Tags.ScreenshotPath); + // repairing the file paths: + if (imagePath.contains("./output")) { + imagePath = imagePath.replace("./output", "../"); + } + write("

State:

"); + write("

"); + write("

Action:

"); + write("

"); + } catch (Exception e) { + System.out.println("ERROR: Adding the Sequence step " + innerLoopCounter + " in the HTML report"); + write("

ERROR Adding current Sequence step " + innerLoopCounter + "

"); + } } - public void addState(State state){ - if(firstStateAdded){ - if(firstActionsAdded){ + public void addState(State state) { + if (firstStateAdded) { + if (firstActionsAdded) { writeStateIntoReport(state); - }else{ + } else { //don't write the state as it is the same - getState is run twice in the beginning, before the first action } - }else{ + } else { firstStateAdded = true; writeStateIntoReport(state); } } - private void writeStateIntoReport(State state){ - try { - String imagePath = state.get(Tags.ScreenshotPath); - if(imagePath.contains("./output")){ - int indexStart = imagePath.indexOf("./output"); - int indexScrn = imagePath.indexOf("scrshots"); - String replaceString = imagePath.substring(indexStart,indexScrn); - imagePath = imagePath.replace(replaceString,"../"); - } - write("

State "+innerLoopCounter+"

"); - write("

concreteID="+state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable")+"

"); - write("

abstractID="+state.get(Tags.AbstractID, "NoAbstractIdAvailable")+"

"); - // try{if(state.get(Tags.Abstract_R_ID)!=null) write("

Abstract_R_ID="+state.get(Tags.Abstract_R_ID)+"

");}catch(Exception e){} - // try{if(state.get(Tags.Abstract_R_T_ID)!=null) write("

Abstract_R_T_ID="+state.get(Tags.Abstract_R_T_ID)+"

");}catch(Exception e){} - // try{if(state.get(Tags.Abstract_R_T_P_ID)!=null) write("

Abstract_R_T_P_ID="+state.get(Tags.Abstract_R_T_P_ID)+"

");}catch(Exception e){} - write("

"); //Smiley face - // file:///E:/TESTAR/TESTAR_dev/testar/target/install/testar/bin/output/output/scrshots/sequence1/SC1padzu12af1193500371.png - // statePath=./output\scrshots\sequence1\SC1y2bsuu2b02920826651.png - }catch(Exception e) { - System.out.println("ERROR: Adding the State number " + innerLoopCounter + " in the HTML report"); - write("

ERROR Adding current State " + innerLoopCounter + "

"); - } - innerLoopCounter++; + private void writeStateIntoReport(State state) { + try { + write("

State " + innerLoopCounter + "

"); + write("

concreteID=" + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + "

"); + write("

abstractID=" + state.get(Tags.AbstractID, "NoAbstractIdAvailable") + "

"); + // try{if(state.get(Tags.Abstract_R_ID)!=null) write("

Abstract_R_ID="+state.get(Tags.Abstract_R_ID)+"

");}catch(Exception e){} + // try{if(state.get(Tags.Abstract_R_T_ID)!=null) write("

Abstract_R_T_ID="+state.get(Tags.Abstract_R_T_ID)+"

");}catch(Exception e){} + // try{if(state.get(Tags.Abstract_R_T_P_ID)!=null) write("

Abstract_R_T_P_ID="+state.get(Tags.Abstract_R_T_P_ID)+"

");}catch(Exception e){} + write("

"); //Smiley face + // file:///E:/TESTAR/TESTAR_dev/testar/target/install/testar/bin/output/output/scrshots/sequence1/SC1padzu12af1193500371.png + // statePath=./output\scrshots\sequence1\SC1y2bsuu2b02920826651.png + } catch (Exception e) { + System.out.println("ERROR: Adding the State number " + innerLoopCounter + " in the HTML report"); + write("

ERROR Adding current State " + innerLoopCounter + "

"); + } + innerLoopCounter++; } - - public void addActions(Set actions){ - if(!firstActionsAdded) firstActionsAdded = true; + public void addActions(Set actions) { + if (!firstActionsAdded) firstActionsAdded = true; write("

Set of actions:

    "); - for(Action action:actions){ + for (Action action : actions) { write("
  • "); // try{if(action.get(Tags.Role)!=null) write("--Role="+action.get(Tags.Role));}catch(Exception e){} // try{if(action.get(Tags.Targets)!=null) write("--Targets="+action.get(Tags.Targets));}catch(Exception e){} - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write(""+ escaped +" || "); - } - }catch(Exception e){} + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + " || "); + } + } catch (Exception e) { + } write(StringEscapeUtils.escapeHtml(action.toString())); - write(" || ConcreteId="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable")); - try{if(action.get(Tags.AbstractID)!=null) write(" || AbstractId="+action.get(Tags.AbstractID));}catch(Exception e){} - try{if(action.get(Tags.Abstract_R_ID)!=null) write(" || Abstract_R_ID="+action.get(Tags.Abstract_R_ID));}catch(Exception e){} - try{if(action.get(Tags.Abstract_R_T_ID)!=null) write(" || Abstract_R_T_ID="+action.get(Tags.Abstract_R_T_ID));}catch(Exception e){} - try{if(action.get(Tags.Abstract_R_T_P_ID)!=null) write(" || Abstract_R_T_P_ID="+action.get(Tags.Abstract_R_T_P_ID));}catch(Exception e){} + write(" || ConcreteId=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable")); + try { + if (action.get(Tags.AbstractID) != null) write(" || AbstractId=" + action.get(Tags.AbstractID)); + } catch (Exception e) { + } + try { + if (action.get(Tags.Abstract_R_ID) != null) + write(" || Abstract_R_ID=" + action.get(Tags.Abstract_R_ID)); + } catch (Exception e) { + } + try { + if (action.get(Tags.Abstract_R_T_ID) != null) + write(" || Abstract_R_T_ID=" + action.get(Tags.Abstract_R_T_ID)); + } catch (Exception e) { + } + try { + if (action.get(Tags.Abstract_R_T_P_ID) != null) + write(" || Abstract_R_T_P_ID=" + action.get(Tags.Abstract_R_T_P_ID)); + } catch (Exception e) { + } write("
    • "); } write("
"); } - public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions){ - if(!firstActionsAdded) firstActionsAdded = true; - if(actions.size()==concreteIdsOfUnvisitedActions.size()){ + public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions) { + if (!firstActionsAdded) firstActionsAdded = true; + if (actions.size() == concreteIdsOfUnvisitedActions.size()) { write("

Set of actions (all unvisited - a new state):

    "); - for(Action action:actions){ + for (Action action : actions) { write("
  • "); - - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write("" + escaped + ""); - } - }catch(Exception e){} - - write(" || ConcreteID="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") - + " || " + StringEscapeUtils.escapeHtml(action.toString())); - + + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + ""); + } + } catch (Exception e) { + } + + write(" || ConcreteID=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + + " || " + StringEscapeUtils.escapeHtml(action.toString())); + write("
    • "); } write("
"); - }else if(concreteIdsOfUnvisitedActions.size()==0){ + } else if (concreteIdsOfUnvisitedActions.size() == 0) { write("

All actions have been visited, set of available actions:

    "); - for(Action action:actions){ - write("
  • "); + for (Action action : actions) { + write("
  • "); - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write("" + escaped + ""); - } - }catch(Exception e){} + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + ""); + } + } catch (Exception e) { + } - write(" || ConcreteID="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") - + " || " + StringEscapeUtils.escapeHtml(action.toString())); + write(" || ConcreteID=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + + " || " + StringEscapeUtils.escapeHtml(action.toString())); - write("
    • "); + write(""); } write("
"); - }else{ - write("

"+concreteIdsOfUnvisitedActions.size()+" out of "+actions.size()+" actions have not been visited yet:

    "); - for(Action action:actions){ - if(concreteIdsOfUnvisitedActions.contains(action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"))){ - //action is unvisited -> showing: - write("
  • "); - - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write("" + escaped + ""); - } - }catch(Exception e){} - - write(" || ConcreteID="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") - + " || " + StringEscapeUtils.escapeHtml(action.toString())); - - write("
    • "); - } + } else { + write("

    " + concreteIdsOfUnvisitedActions.size() + " out of " + actions.size() + " actions have not been visited yet:

      "); + for (Action action : actions) { + if (concreteIdsOfUnvisitedActions.contains(action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"))) { + //action is unvisited -> showing: + write("
    • "); + + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + ""); + } + } catch (Exception e) { + } + + write(" || ConcreteID=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + + " || " + StringEscapeUtils.escapeHtml(action.toString())); + + write("
      • "); + } } write("
    "); } } - public void addSelectedAction(State state, Action action){ - String screenshotDir = OutputStructure.screenshotsOutputDir; -// System.out.println("path="+state_path); - if(screenshotDir.contains("./output")){ - int indexStart = screenshotDir.indexOf("./output"); - int indexScrn = screenshotDir.indexOf("scrshots"); - String replaceString = screenshotDir.substring(indexStart,indexScrn); - screenshotDir = screenshotDir.replace(replaceString,"../"); + public void addSelectedAction(State state, Action action) { + write("

    Selected Action " + innerLoopCounter + " leading to State " + innerLoopCounter + "\"

    "); + write("

    concreteID=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable")); + + if (action.get(Tags.Desc, null) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write(" || " + escaped); } -// System.out.println("path="+actionPath); - String actionPath = screenshotDir + File.separator - + OutputStructure.startInnerLoopDateString + "_" + OutputStructure.executedSUTname - + "_sequence_" + OutputStructure.sequenceInnerLoopCount - + File.separator + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + "_" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + ".png"; -// System.out.println("path="+actionPath); - write("

    Selected Action "+innerLoopCounter+" leading to State "+innerLoopCounter+"\"

    "); - write("

    concreteID="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable")); - - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write(" || "+ escaped); - } - }catch(Exception e){} write("

    "); - if(actionPath.contains("./output")){ - actionPath = actionPath.replace("./output",".."); + write("

    "); //Smiley face + } + + public void addTestVerdict(Verdict verdict) { + String verdictInfo = verdict.info(); + if (verdict.severity() > Verdict.OK.severity()) + verdictInfo = verdictInfo.replace(Verdict.OK.info(), ""); + + write("

    Test verdict for this sequence: " + verdictInfo + "

    "); + write("

    Severity: " + verdict.severity() + "

    "); + } + + @Override + public void addVisualValidationResult(MatcherResult result, State state, @Nullable Action action) { + if (result != null) { + if (!result.getNoLocationMatches().isEmpty() || !result.getResult().isEmpty()) { + write("

    Visual validation result:

    "); + + // Add the annotated screenshot + StringBuilder screenshotPath = new StringBuilder(action != null ? + getActionScreenshotPath(state, action) : getScreenshotPath(state)); + screenshotPath.insert(screenshotPath.indexOf(".png"), MatcherResult.ScreenshotPostFix); + write("

    "); + + // List the expected elements without a location match. + result.getNoLocationMatches().forEach(it -> + write("

    No match for: \"" + it._text + "\" at location: " + it._location + "

    ") + ); + + // List the expected elements with a location match. + composeMatchedResultTable(result); + } + } else { + write("

    Visual validation result:

    "); + write("

    No results available.

    "); + } + } + + private void composeMatchedResultTable(MatcherResult result) { + + for (ContentMatchResult contentResult : result.getResult()) { + // Based on the location try lookup the OCR matched entries. + Optional locationMatch = result.getLocationMatches().stream() + .filter(it -> it.location.location == contentResult.foundLocation).findFirst(); + if (locationMatch.isPresent()) { + LocationMatch it = locationMatch.get(); + write("
    "); + write("
    Expect: \"" + it.expectedElement._text + "\" at location: " + it.location.location + "\" location matched with:
    "); + it.recognizedElements.forEach(recognizedElement -> + write("
    \"" + recognizedElement._text + "\" at location: " + recognizedElement._location + "\" confidence: " + recognizedElement._confidence + "
    ")); + write("
    "); + } + + // Write content match result. + write("" + ); + writeTableRow(() -> { + writeTableHeader("Result:"); + contentResult.expectedResult.getResult().forEach(it -> + writeTableCell(it.getMatchResult()) + ); + return true; + }); + writeTableRow(() -> { + writeTableHeader("Expected:"); + contentResult.expectedResult.getResult().forEach(it -> + writeTableCell(it.getCharacterMatch().getCharacter()) + ); + return true; + }); + writeTableRow(() -> { + writeTableHeader("Found:"); + contentResult.expectedResult.getResult().forEach(it -> + { + CharacterMatchEntry tmp = it.getCharacterMatch(); + writeTableCell(tmp.isMatched() ? tmp.getCounterPart().getCharacter() : ""); + } + ); + return true; + }); + write("
    \"" + contentResult.expectedText + "\" " + + "[" + contentResult.totalMatched + "/" + contentResult.totalExpected + "]
    "); + if (contentResult.recognizedResult.getResult().stream().anyMatch(CharacterMatchEntry::isNotMatched)) { + write("

    No match for OCR items:

    "); + write("
      "); + contentResult.recognizedResult.getResult().forEach(it -> { + if (it.isNotMatched()) { + write("
    • " + it.getCharacter() + "
      • "); + } + }); + write("
    "); + } } - write("

    "); //Smiley face } - public void addTestVerdict(Verdict verdict){ - String verdictInfo = verdict.info(); - if(verdict.severity() > Verdict.OK.severity()) - verdictInfo = verdictInfo.replace(Verdict.OK.info(), ""); - - write("

    Test verdict for this sequence: "+verdictInfo+"

    "); - write("

    Severity: "+verdict.severity()+"

    "); + private void writeTableHeader(T data) { + write("" + data + ""); + } + + private void writeTableCell(T data) { + write("" + data + ""); + } + + private void writeTableRow(Supplier data) { + write(""); + data.get(); + write(""); } - public void close() { - for(String s:HTMLReporter.FOOTER){ + for (String s : HTMLReporter.FOOTER) { write(s); } out.close(); diff --git a/testar/src/nl/ou/testar/HtmlReporting/Reporting.java b/testar/src/nl/ou/testar/HtmlReporting/Reporting.java index 2eade1995..21743f00a 100644 --- a/testar/src/nl/ou/testar/HtmlReporting/Reporting.java +++ b/testar/src/nl/ou/testar/HtmlReporting/Reporting.java @@ -30,6 +30,8 @@ package nl.ou.testar.HtmlReporting; +import nl.ou.testar.visualvalidation.matcher.MatcherResult; +import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.State; import java.util.Set; import org.fruit.alayer.Action; @@ -42,5 +44,6 @@ public interface Reporting { public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions); public void addSelectedAction(State state, Action action); public void addTestVerdict(Verdict verdict); + void addVisualValidationResult(MatcherResult result, State state, @Nullable Action action); public void close(); } diff --git a/testar/src/nl/ou/testar/HtmlReporting/XMLSequenceReport.java b/testar/src/nl/ou/testar/HtmlReporting/XMLSequenceReport.java index 1be064355..868eff57f 100644 --- a/testar/src/nl/ou/testar/HtmlReporting/XMLSequenceReport.java +++ b/testar/src/nl/ou/testar/HtmlReporting/XMLSequenceReport.java @@ -35,6 +35,8 @@ import java.io.PrintWriter; import java.util.Set; +import nl.ou.testar.visualvalidation.matcher.MatcherResult; +import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.Action; import org.fruit.alayer.State; import org.fruit.alayer.Verdict; @@ -154,6 +156,11 @@ public void addTestVerdict(Verdict verdict) { out.flush(); } + @Override + public void addVisualValidationResult(MatcherResult result, State state, @Nullable Action action) { + + } + private TestRun testRun = new TestRun(); @Override diff --git a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java index 37e65ab45..71671fdb5 100644 --- a/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/DummyVisualValidator.java @@ -1,5 +1,6 @@ package nl.ou.testar.visualvalidation; +import nl.ou.testar.visualvalidation.matcher.MatcherResult; import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.AWTCanvas; import org.fruit.alayer.State; @@ -7,13 +8,13 @@ public class DummyVisualValidator implements VisualValidationManager { @Override - public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { - + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { + return new MatcherResult(); } @Override - public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { - + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { + return new MatcherResult(); } @Override diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java index 1b325b078..dd879f1c8 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationManager.java @@ -1,5 +1,6 @@ package nl.ou.testar.visualvalidation; +import nl.ou.testar.visualvalidation.matcher.MatcherResult; import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.AWTCanvas; import org.fruit.alayer.State; @@ -10,16 +11,18 @@ public interface VisualValidationManager { * Analyze the captured image and update the verdict. * @param state The state of the application. * @param screenshot The captured screenshot of the current state. + * @return The matching result of the expected text and the detected text on the captured image. */ - void AnalyzeImage(State state, @Nullable AWTCanvas screenshot); + MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot); /** * Analyze the captured image and update the verdict. * @param state The state of the application. * @param screenshot The captured screenshot of the current state. * @param widget Optional, the corresponding widget when the screenshot is an action shot. + * @return The matching result of the expected text and the detected text on the captured image. */ - void AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget); + MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget); /** * Destroy the visual validation manager. diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationTag.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationTag.java new file mode 100644 index 000000000..4cfed0c5d --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationTag.java @@ -0,0 +1,9 @@ +package nl.ou.testar.visualvalidation; + +import org.fruit.alayer.Tag; +import org.fruit.alayer.TagsBase; +import org.fruit.alayer.Verdict; + +public class VisualValidationTag extends TagsBase { + public static final Tag VisualValidationVerdict = from("VisualValidationVerdict", Verdict.class); +} diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidationVerdict.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidationVerdict.java new file mode 100644 index 000000000..fe825ca18 --- /dev/null +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidationVerdict.java @@ -0,0 +1,31 @@ +package nl.ou.testar.visualvalidation; + +import nl.ou.testar.visualvalidation.matcher.ContentMatchResult; +import org.fruit.alayer.Verdict; + +public class VisualValidationVerdict { + static final double warningSeverity = 0.1; + static final double errorSeverity = 0.15; + static final double failedToMatchSeverity = 0.16; + + private static String composeVerdictMessage(ContentMatchResult match) { + return String.format("\"%s\" matched %d%%.", match.expectedText, match.matchedPercentage); + } + + public static Verdict createSuccessVerdict(ContentMatchResult match) { + return new Verdict(Verdict.SEVERITY_OK, composeVerdictMessage(match)); + } + + public static Verdict createAlmostMatchedVerdict(ContentMatchResult match) { + return new Verdict(warningSeverity, composeVerdictMessage(match)); + } + + public static Verdict createHardlyMatchedVerdict(ContentMatchResult match) { + return new Verdict(errorSeverity, composeVerdictMessage(match)); + } + + public static Verdict createFailedToMatchVerdict(TextElement match) { + return new Verdict(failedToMatchSeverity, String.format("Failed to match \"%s\".", match._text)); + } + +} diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 772135f3c..819859768 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -1,9 +1,11 @@ package nl.ou.testar.visualvalidation; +import es.upv.staq.testar.serialisation.ScreenshotSerialiser; import nl.ou.testar.visualvalidation.extractor.ExpectedElement; import nl.ou.testar.visualvalidation.extractor.ExpectedTextCallback; import nl.ou.testar.visualvalidation.extractor.ExtractorFactory; import nl.ou.testar.visualvalidation.extractor.TextExtractorInterface; +import nl.ou.testar.visualvalidation.matcher.ContentMatchResult; import nl.ou.testar.visualvalidation.matcher.MatcherResult; import nl.ou.testar.visualvalidation.matcher.VisualMatcher; import nl.ou.testar.visualvalidation.matcher.VisualMatcherFactory; @@ -16,40 +18,37 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.fruit.alayer.AWTCanvas; -import org.fruit.alayer.AbsolutePosition; -import org.fruit.alayer.Color; -import org.fruit.alayer.FillPattern; -import org.fruit.alayer.Pen; +import org.fruit.alayer.Action; import org.fruit.alayer.State; -import org.fruit.alayer.StrokePattern; +import org.fruit.alayer.Tags; import org.fruit.alayer.Verdict; import org.fruit.alayer.Widget; -import org.fruit.alayer.visualizers.TextVisualizer; import org.testar.Logger; +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.Rectangle; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import static nl.ou.testar.visualvalidation.VisualValidationTag.VisualValidationVerdict; +import static nl.ou.testar.visualvalidation.VisualValidationVerdict.*; + public class VisualValidator implements VisualValidationManager, OcrResultCallback, ExpectedTextCallback { + static final Color darkOrange = new Color(255, 128, 0); private final String TAG = "VisualValidator"; - private int analysisId = 0; - private final VisualMatcher _matcher; - private MatcherResult _matcherResult = null; - private final OcrEngineInterface _ocrEngine; private final Object _ocrResultSync = new Object(); private final AtomicBoolean _ocrResultReceived = new AtomicBoolean(); - private List _ocrItems = null; - private final TextExtractorInterface _extractor; private final Object _expectedTextSync = new Object(); private final AtomicBoolean _expectedTextReceived = new AtomicBoolean(); + private int analysisId = 0; + private MatcherResult _matcherResult = null; + private List _ocrItems = null; private List _expectedText = null; - protected final static Pen RedPen = Pen.newPen().setColor(Color.Red). - setFillPattern(FillPattern.None).setStrokePattern(StrokePattern.Solid).build(); - public VisualValidator(@NonNull VisualValidationSettings settings) { OcrConfiguration ocrConfig = settings.ocrConfiguration; if (ocrConfig.enabled) { @@ -67,28 +66,36 @@ public VisualValidator(@NonNull VisualValidationSettings settings) { _matcher = VisualMatcherFactory.createLocationMatcher(settings.matcherConfiguration); } + public static boolean isBetween(int x, int lower, int upper) { + return lower <= x && x <= upper; + } + @Override - public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { - AnalyzeImage(state, screenshot, null); + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { + return AnalyzeImage(state, screenshot, null); } @Override - public void AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { // Create new session startNewAnalysis(); - // Start ocr analysis, provide callback once finished. - parseScreenshot(screenshot); - - // Start extracting text, provide callback once finished. - extractExpectedText(state, widget); + if (screenshot != null) { + // Start ocr analysis, provide callback once finished. + parseScreenshot(screenshot); - // Match the expected text with the detected text. - matchText(); + // Start extracting text, provide callback once finished. + extractExpectedText(state, widget); - updateVerdict(state); + // Match the expected text with the detected text. + matchText(); - storeAnalysis(); + // Calculate the verdict and create a screenshot with annotations which can be used by the HTML reporter. + processMatchResult(state, screenshot); + } else { + Logger.log(Level.ERROR, TAG, "No screenshot for current state, skipping visual validation"); + } + return _matcherResult; } private void startNewAnalysis() { @@ -103,13 +110,8 @@ private void startNewAnalysis() { Logger.log(Level.INFO, TAG, "Starting new analysis {}", analysisId); } - private void parseScreenshot(@Nullable AWTCanvas screenshot) { - if (screenshot != null) { - _ocrEngine.AnalyzeImage(screenshot.image(), this); - - } else { - Logger.log(Level.ERROR, TAG, "No screenshot for current state"); - } + private void parseScreenshot(AWTCanvas screenshot) { + _ocrEngine.AnalyzeImage(screenshot.image(), this); } private void extractExpectedText(State state, @Nullable Widget widget) { @@ -140,26 +142,72 @@ private void waitForResults() { waitForResult(_expectedTextReceived, _expectedTextSync); } - private void updateVerdict(State state) { + private void processMatchResult(State state, AWTCanvas screenshot) { + // Create a copy, so we can annotate the results without modifying the original one. + final AWTCanvas copy = new AWTCanvas( + screenshot.width(), + screenshot.height(), + screenshot.deepCopyImage(), + AWTCanvas.StorageFormat.PNG, + 1.0); + final Verdict[] validationVerdict = {Verdict.OK}; if (_matcherResult != null) { - // Analysis the raw result and create a verdict. - _matcherResult.getResult().forEach(result -> - Logger.log(Level.INFO, TAG, "Content match result: {}", result) + // Draw a rectangle on each expected text element that has been recognized. + final ContentMatchResult[] lowestMatch = {null}; + _matcherResult.getResult().forEach(result -> { + java.awt.Color penColor; + Verdict verdict; + if (result.matchedPercentage == 100) { + penColor = java.awt.Color.green; + verdict = createSuccessVerdict(result); + } else if (isBetween(result.matchedPercentage, 75, 99)) { + penColor = java.awt.Color.yellow; + verdict = createAlmostMatchedVerdict(result); + } else { + penColor = darkOrange; + verdict = createHardlyMatchedVerdict(result); + } + + // Store the result with the lowest percentage including the corresponding verdict. + if ((lowestMatch[0] == null) || + (lowestMatch[0].matchedPercentage > result.matchedPercentage)) { + lowestMatch[0] = result; + validationVerdict[0] = verdict; + } + + drawRectangle(copy, result.foundLocation, penColor); + } ); - - Verdict result = new Verdict(Verdict.SEVERITY_WARNING, "Not all texts has been recognized", - new TextVisualizer(new AbsolutePosition(10, 10), "->", RedPen)); - + _matcherResult.getNoLocationMatches().forEach(result -> + { + drawRectangle(copy, result._location, java.awt.Color.red); + // Overwrite the verdict if we couldn't find a location match for expected text. + validationVerdict[0] = createFailedToMatchVerdict(result); + } + ); + } + // Store the validation verdict for this run. + state.set(VisualValidationVerdict, validationVerdict[0]); + + // Store the annotated screenshot, so they can be used by the HTML report generator. + String stateId = state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"); + Action action = state.get(Tags.ExecutedAction, null); + if (action != null) { + String actionID = action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"); + ScreenshotSerialiser.saveActionshot(stateId, actionID + MatcherResult.ScreenshotPostFix, copy); } else { - // Set verdict to failure we should have a matcher result as minimal input. - Logger.log(Level.INFO, TAG, "No result"); + ScreenshotSerialiser.saveStateshot(stateId + MatcherResult.ScreenshotPostFix, copy); } - Logger.log(Level.INFO, TAG, "Updating verdict {}"); + + Logger.log(Level.INFO, TAG, "Processed match results"); } - private void storeAnalysis() { - Logger.log(Level.INFO, TAG, "Storing analysis"); + private void drawRectangle(AWTCanvas screenshot, Rectangle location, java.awt.Color color) { + Graphics2D graphics = screenshot.image().createGraphics(); + graphics.setColor(color); + graphics.drawRect(location.x, location.y, location.width - 1, location.height - 1); + graphics.dispose(); } @Override diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java index a82278bef..2f8da6485 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatch.java @@ -16,6 +16,20 @@ public CharacterMatch(char character){ this.character = new CharacterMatchEntry(character); } + /** + * @return Get the result of the character match. + */ + public CharacterMatchResult getMatchResult(){ + return result; + } + + /** + * @return Get the character match. + */ + public CharacterMatchEntry getCharacterMatch(){ + return character; + } + @Override public String toString() { return "CharacterMatch{" + diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java index b6cb74a68..4ed4aac8b 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchEntry.java @@ -3,7 +3,7 @@ import org.checkerframework.checker.nullness.qual.NonNull; /** - * Container to store the matched counter part of the expected character. + * Container to store the matched counterpart of the expected character. */ public class CharacterMatchEntry { final Character character; @@ -29,6 +29,20 @@ public void Match(@NonNull CharacterMatchEntry matchedCharacter) { matchedCharacter.match = this; } + /** + * @return Get the character. + */ + public Character getCharacter() { + return character; + } + + /** + * @return Get the matched counterpart of this character. + */ + public CharacterMatchEntry getCounterPart(){ + return match; + } + /** * Check if the character has not matched. * diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java index ebe28282e..fbae7b0b2 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/CharacterMatchResult.java @@ -3,9 +3,25 @@ /** * The match result of an individual character. */ -enum CharacterMatchResult { +public enum CharacterMatchResult { MATCHED, CASE_MISMATCH, WHITESPACE_CORRECTED, - NO_MATCH + NO_MATCH; + + @Override + public String toString() { + switch (this) { + case MATCHED: + return "V"; + case CASE_MISMATCH: + return "C"; + case WHITESPACE_CORRECTED: + return "W"; + case NO_MATCH: + return "X"; + default: + return ""; + } + } } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java index 276cd7684..e0a079062 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatchResult.java @@ -16,6 +16,7 @@ public class ContentMatchResult implements Comparable { public final Location foundLocation; public final long totalMatched; public final long totalExpected; + public final int matchedPercentage; public ContentMatchResult(ExpectedTextMatchResult expectedResult, RecognizedTextMatchResult recognizedResult, Location foundLocation) { this.expectedResult = expectedResult; @@ -25,6 +26,7 @@ public ContentMatchResult(ExpectedTextMatchResult expectedResult, RecognizedText .filter(e -> e.result != CharacterMatchResult.NO_MATCH) .count(); totalExpected = expectedResult.expectedText.size(); + matchedPercentage = Math.round((float) totalMatched * 100 / totalExpected); expectedText = expectedResult.expectedText.stream(). map(e -> e.character.character).collect(Collector.of( StringBuilder::new, @@ -42,18 +44,7 @@ public String toString() { .append("\"\n") .append("Result: ") - .append(expectedResult.expectedText.stream().map(it -> { - switch (it.result) { - case WHITESPACE_CORRECTED: - return "W"; - case CASE_MISMATCH: - return "C"; - case MATCHED: - return "V"; - default: - return "X"; - } - }).collect(Collectors.toList())) + .append(expectedResult.expectedText.stream().map(it -> it.result).collect(Collectors.toList())) .append(" [").append(totalMatched).append("/").append(totalExpected).append("]\n") .append("Expect: ").append(expectedResult.expectedText.stream() diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java index 2698b0935..6024df1f1 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ContentMatcher.java @@ -51,7 +51,7 @@ public static ContentMatchResult Match(LocationMatch locationMatch, MatcherConfi for (int k = indexCounter; k < unMatchedSize; k++) { CharacterMatchEntry item = recognizedResult.recognized.get(k); - // Try to match the actual char case sensitive: + // Try to match the actual char case-sensitive: if (item.character == actual && item.isNotMatched()) { expectedChar.character.Match(item); expectedChar.result = CharacterMatchResult.MATCHED; @@ -60,7 +60,7 @@ public static ContentMatchResult Match(LocationMatch locationMatch, MatcherConfi break; } - // Try to match the actual char as case insensitive: + // Try to match the actual char as case-insensitive: if (Character.isLetter(actual)) { int CASING = 32; if (Math.abs(item.character.compareTo(actual)) == CASING && item.isNotMatched()) { @@ -76,7 +76,7 @@ public static ContentMatchResult Match(LocationMatch locationMatch, MatcherConfi correctWhitespaces(expectedResult); - return new ContentMatchResult(expectedResult, recognizedResult); + return new ContentMatchResult(expectedResult, recognizedResult, locationMatch.expectedElement._location); } @NonNull @@ -88,7 +88,7 @@ static List sortRecognizedElements(LocationMatch locationMatc // Get the sorted Y-axe coordinates for the identified lines. List bucketSorted = lineBucket.keySet().stream().sorted().collect(Collectors.toList()); - // For each line sort the elements based on their X-axis coordinate and add the to the result. + // For each line sort the elements based on their X-axis coordinate and add to the result. List sorted = new ArrayList<>(); bucketSorted.forEach(line -> { @@ -110,7 +110,7 @@ static Map> sortRecognizedElementsPerTextLine(L .average() .orElse(Double.NaN); - // Create a overview of pairs reflecting the center line of the text and the recognized element. + // Create an overview of pairs reflecting the center line of the text and the recognized element. List> lines = locationMatch.recognizedElements.stream() .map(e -> new Pair<>(e._location.y + (e._location.height / 2), e)) .collect(Collectors.toList()); diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java index bdb1d3b7a..2c6273009 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/ExpectedTextMatchResult.java @@ -12,15 +12,23 @@ public class ExpectedTextMatchResult { /** * Constructor. + * * @param expectedText The expected text that we are matching. */ public ExpectedTextMatchResult(String expectedText) { this.expectedText = new ArrayList<>(expectedText.length()); - for (char ch: expectedText.toCharArray()) { + for (char ch : expectedText.toCharArray()) { this.expectedText.add(new CharacterMatch(ch)); } } + /** + * @return The matching result of the expected text characters. + */ + public ArrayList getResult() { + return expectedText; + } + @Override public String toString() { return "ExpectedTextMatchResult (" + diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java index 8fd682a0a..6cdcc40bf 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherResult.java @@ -10,6 +10,12 @@ public class MatcherResult { private final Set locationMatches = new TreeSet<>(); private final Set contentMatchResults = new TreeSet<>(); + /** + * Postfix for the screenshots created by the visual validation module. + * Should be added to the end of normal action and state screenshots. E.g. "filename+postfix.extension" + */ + public static final String ScreenshotPostFix = "_vv"; + public void addContentMatchResult(ContentMatchResult result) { contentMatchResults.add(result); } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java b/testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java index 4b6650c3b..484341542 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/RecognizedTextMatchResult.java @@ -23,6 +23,13 @@ public RecognizedTextMatchResult(List recognizedElements) { .collect(Collectors.toList()); } + /** + * @return The matching result of the recognized text characters. + */ + public List getResult() { + return recognized; + } + @Override public String toString() { return "RecognizedTextMatchResult{" + diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index 02e682319..49ed1e422 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -54,7 +54,6 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintStream; -import java.net.URI; import java.net.URISyntaxException; import java.util.*; import java.util.List; @@ -65,15 +64,15 @@ import javax.swing.*; import javax.swing.event.HyperlinkEvent; -import javax.swing.event.HyperlinkListener; import es.upv.staq.testar.*; import nl.ou.testar.*; import nl.ou.testar.HtmlReporting.Reporting; -import nl.ou.testar.visualvalidation.VisualValidationFactory; -import nl.ou.testar.visualvalidation.VisualValidationManager; import nl.ou.testar.StateModel.StateModelManager; import nl.ou.testar.StateModel.StateModelManagerFactory; +import nl.ou.testar.visualvalidation.VisualValidationFactory; +import nl.ou.testar.visualvalidation.VisualValidationManager; +import nl.ou.testar.visualvalidation.VisualValidationTag; import org.fruit.Assert; import org.fruit.Pair; import org.fruit.Util; @@ -1552,15 +1551,20 @@ private void setStateScreenshot(State state) { screenshot = ProtocolUtil.getStateshotBinary(state); } String screenshotPath = ScreenshotSerialiser.saveStateshot(state.get(Tags.ConcreteIDCustom, - "NoConcreteIdAvailable"), screenshot); - state.set(Tags.ScreenshotPath, screenshotPath); - } - visualValidationManager.AnalyzeImage(state, screenshot); - } + "NoConcreteIdAvailable"), screenshot); + state.set(Tags.ScreenshotPath, screenshotPath); + } - @Override + htmlReport.addVisualValidationResult( + visualValidationManager.AnalyzeImage(state, screenshot), state, null + ); + Logger.log(org.apache.logging.log4j.Level.DEBUG, "TESTING", "Continued"); + } + + @Override protected Verdict getVerdict(State state){ Assert.notNull(state); + Verdict visualValidationVerdict = state.get(VisualValidationTag.VisualValidationVerdict, Verdict.OK); //------------------- // ORACLES FOR FREE //------------------- @@ -1581,21 +1585,22 @@ protected Verdict getVerdict(State state){ this.suspiciousTitlesPattern = Pattern.compile(settings().get(ConfigTags.SuspiciousTitles), Pattern.UNICODE_CHARACTER_CLASS); // search all widgets for suspicious String Values - Verdict suspiciousValueVerdict = Verdict.OK; + Verdict suspiciousValueVerdict; for(Widget w : state) { suspiciousValueVerdict = suspiciousStringValueMatcher(w); if(suspiciousValueVerdict.severity() == Verdict.SEVERITY_SUSPICIOUS_TITLE) { - return suspiciousValueVerdict; + return suspiciousValueVerdict.join(visualValidationVerdict); } } if (this.nonSuitableAction){ this.nonSuitableAction = false; - return new Verdict(Verdict.SEVERITY_WARNING, "Non suitable action for state"); + return new Verdict(Verdict.SEVERITY_WARNING, "Non suitable action for state") + .join(visualValidationVerdict); } // if everything was OK ... - return Verdict.OK; + return Verdict.OK.join(visualValidationVerdict); } private Verdict suspiciousStringValueMatcher(Widget w) { @@ -1736,15 +1741,20 @@ else if (actions.isEmpty()){ //TODO move the CPU metric to another helper class that is not default "TrashBinCode" or "SUTprofiler" //TODO check how well the CPU usage based waiting works protected boolean executeAction(SUT system, State state, Action action){ - AWTCanvas screenshot = null; - if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)){ - screenshot = WdProtocolUtil.getActionshot(state,action); - }else{ - screenshot = ProtocolUtil.getActionshot(state,action); - } - ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), screenshot); - - visualValidationManager.AnalyzeImage(state, screenshot, action.get(OriginWidget, null)); + AWTCanvas screenshot; + if (NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { + screenshot = WdProtocolUtil.getActionshot(state, action); + } else { + screenshot = ProtocolUtil.getActionshot(state, action); + } + state.set(ExecutedAction, action); + ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), screenshot); + + htmlReport.addVisualValidationResult( + visualValidationManager.AnalyzeImage(state, screenshot, action.get(OriginWidget, null)), + state, action + ); + Logger.log(org.apache.logging.log4j.Level.DEBUG, "TESTING", "Continued action"); double waitTime = settings.get(ConfigTags.TimeToWaitAfterAction); diff --git a/testar/src/org/testar/protocols/DesktopProtocol.java b/testar/src/org/testar/protocols/DesktopProtocol.java index ea99bc868..4a086b5cc 100644 --- a/testar/src/org/testar/protocols/DesktopProtocol.java +++ b/testar/src/org/testar/protocols/DesktopProtocol.java @@ -57,7 +57,6 @@ public class DesktopProtocol extends GenericUtilsProtocol { //Attributes for adding slide actions protected static double SCROLL_ARROW_SIZE = 36; // sliding arrows protected static double SCROLL_THICK = 16; //scroll thickness - protected Reporting htmlReport; protected State latestState; /** diff --git a/testar/src/org/testar/protocols/WebdriverProtocol.java b/testar/src/org/testar/protocols/WebdriverProtocol.java index 4ba906046..51258941b 100644 --- a/testar/src/org/testar/protocols/WebdriverProtocol.java +++ b/testar/src/org/testar/protocols/WebdriverProtocol.java @@ -82,7 +82,6 @@ public class WebdriverProtocol extends GenericUtilsProtocol { //Attributes for adding slide actions protected static double SCROLL_ARROW_SIZE = 36; // sliding arrows protected static double SCROLL_THICK = 16; //scroll thickness - protected Reporting htmlReport; protected State latestState; protected static Set existingCssClasses = new HashSet<>(); From e701dea5cfa412480a1c16ce6f3668cdc37fbe16 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 8 Sep 2021 13:24:35 +0200 Subject: [PATCH 29/40] Replaced hard-coded threshold with configuration threshold. Fixed sequence abortion on visual mismatch. --- .../ou/testar/HtmlReporting/HtmlSequenceReport.java | 2 +- .../ou/testar/visualvalidation/VisualValidator.java | 12 ++++++++---- .../matcher/MatcherConfiguration.java | 8 +++++++- testar/src/org/fruit/monkey/DefaultProtocol.java | 8 +++++--- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java b/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java index 5657f21f5..68e0e763a 100644 --- a/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java +++ b/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java @@ -372,7 +372,7 @@ private void composeMatchedResultTable(MatcherResult result) { // Write content match result. write("" + "[" + contentResult.totalMatched + "/" + contentResult.totalExpected + "] " + contentResult.matchedPercentage +"%" ); writeTableRow(() -> { writeTableHeader("Result:"); diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 819859768..f644be012 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -44,26 +44,28 @@ public class VisualValidator implements VisualValidationManager, OcrResultCallba private final TextExtractorInterface _extractor; private final Object _expectedTextSync = new Object(); private final AtomicBoolean _expectedTextReceived = new AtomicBoolean(); + private final VisualValidationSettings _settings; private int analysisId = 0; private MatcherResult _matcherResult = null; private List _ocrItems = null; private List _expectedText = null; public VisualValidator(@NonNull VisualValidationSettings settings) { - OcrConfiguration ocrConfig = settings.ocrConfiguration; + _settings = settings; + OcrConfiguration ocrConfig = _settings.ocrConfiguration; if (ocrConfig.enabled) { _ocrEngine = OcrEngineFactory.createOcrEngine(ocrConfig); } else { _ocrEngine = null; } - if (settings.protocol.contains("webdriver_generic")) { + if (_settings.protocol.contains("webdriver_generic")) { _extractor = ExtractorFactory.CreateExpectedTextExtractorWebdriver(); } else { _extractor = ExtractorFactory.CreateExpectedTextExtractorDesktop(); } - _matcher = VisualMatcherFactory.createLocationMatcher(settings.matcherConfiguration); + _matcher = VisualMatcherFactory.createLocationMatcher(_settings.matcherConfiguration); } public static boolean isBetween(int x, int lower, int upper) { @@ -161,7 +163,9 @@ private void processMatchResult(State state, AWTCanvas screenshot) { if (result.matchedPercentage == 100) { penColor = java.awt.Color.green; verdict = createSuccessVerdict(result); - } else if (isBetween(result.matchedPercentage, 75, 99)) { + } else if (isBetween(result.matchedPercentage, + _settings.matcherConfiguration.failedToMatchPercentageThreshold, + 99)) { penColor = java.awt.Color.yellow; verdict = createAlmostMatchedVerdict(result); } else { diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java index b34960a16..c81e69a58 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/MatcherConfiguration.java @@ -4,11 +4,13 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; @XmlAccessorType(XmlAccessType.FIELD) public class MatcherConfiguration extends ExtendedSettingBase { public Integer locationMatchMargin; boolean loggingEnabled; + public Integer failedToMatchPercentageThreshold; @Override public String toString() { @@ -21,13 +23,17 @@ public static MatcherConfiguration CreateDefault() { MatcherConfiguration instance = new MatcherConfiguration(); instance.locationMatchMargin = 0; instance.loggingEnabled = false; + instance.failedToMatchPercentageThreshold = 75; return instance; } @Override public int compareTo(MatcherConfiguration other) { int result = -1; - if (locationMatchMargin.equals(other.locationMatchMargin) && (loggingEnabled == other.loggingEnabled)) { + if (locationMatchMargin.equals(other.locationMatchMargin) && + (loggingEnabled == other.loggingEnabled) && + (Objects.equals(failedToMatchPercentageThreshold, other.failedToMatchPercentageThreshold)) + ) { result = 0; } return result; diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index 49ed1e422..04f349cc6 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -105,6 +105,7 @@ import org.openqa.selenium.SessionNotCreatedException; import org.testar.Logger; import org.testar.OutputStructure; +import org.testar.settings.ExtendedSettingsFactory; public class DefaultProtocol extends RuntimeControlsProtocol { @@ -1558,7 +1559,6 @@ private void setStateScreenshot(State state) { htmlReport.addVisualValidationResult( visualValidationManager.AnalyzeImage(state, screenshot), state, null ); - Logger.log(org.apache.logging.log4j.Level.DEBUG, "TESTING", "Continued"); } @Override @@ -1754,7 +1754,6 @@ protected boolean executeAction(SUT system, State state, Action action){ visualValidationManager.AnalyzeImage(state, screenshot, action.get(OriginWidget, null)), state, action ); - Logger.log(org.apache.logging.log4j.Level.DEBUG, "TESTING", "Continued action"); double waitTime = settings.get(ConfigTags.TimeToWaitAfterAction); @@ -1852,7 +1851,10 @@ protected String getRandomText(Widget w){ * @return */ protected boolean moreActions(State state) { - return (!settings().get(ConfigTags.StopGenerationOnFault) || !faultySequence) && + // When visual validation module is enabled continue even when faults are detected. + boolean suppressFaultySequence = ExtendedSettingsFactory.createVisualValidationSettings().enabled || + (!settings().get(ConfigTags.StopGenerationOnFault) || !faultySequence); + return suppressFaultySequence && state.get(Tags.IsRunning, false) && !state.get(Tags.NotResponding, false) && //actionCount() < settings().get(ConfigTags.SequenceLength) && actionCount() <= lastSequenceActionNumber && From d557c4790a8c20a3c604d6e62a9cb5e4f796049c Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Wed, 15 Sep 2021 16:49:21 +0200 Subject: [PATCH 30/40] Workaround for supporting Qt desktop applications. --- .../org/fruit/alayer/windows/StateFetcher.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/windows/src/org/fruit/alayer/windows/StateFetcher.java b/windows/src/org/fruit/alayer/windows/StateFetcher.java index ae7f3f410..ff3c58333 100644 --- a/windows/src/org/fruit/alayer/windows/StateFetcher.java +++ b/windows/src/org/fruit/alayer/windows/StateFetcher.java @@ -327,7 +327,7 @@ private UIAElement uiaDescend(long hwnd, long uiaCachePointer, UIAElement parent long uiaWindowPointer = Windows.IUIAutomationElement_GetPattern(uiaCachePointer, Windows.UIA_WindowPatternId, true); if(uiaWindowPointer != 0){ uiaElement.wndInteractionState = Windows.IUIAutomationWindowPattern_get_WindowInteractionState(uiaWindowPointer, true); - uiaElement.blocked = (uiaElement.wndInteractionState != Windows.WindowInteractionState_ReadyForUserInteraction); + uiaElement.blocked = isElementBlocked(uiaElement); uiaElement.isTopmostWnd = Windows.IUIAutomationWindowPattern_get_IsTopmost(uiaWindowPointer, true); uiaElement.isModal = Windows.IUIAutomationWindowPattern_get_IsModal(uiaWindowPointer, true); @@ -508,7 +508,18 @@ private UIAElement uiaDescend(long hwnd, long uiaCachePointer, UIAElement parent return modalElement; } - + + private boolean isElementBlocked(UIAElement uiaElement) { + // Qt applications are always started in the running state. + // Without this dedicated check TESTAR can't find any actions for the Qt application. + if (Objects.equals(uiaElement.frameworkId, "Qt")) { + return !(uiaElement.wndInteractionState == Windows.WindowInteractionState_ReadyForUserInteraction || + uiaElement.wndInteractionState == Windows.WindowInteractionState_Running); + } else { + return (uiaElement.wndInteractionState != Windows.WindowInteractionState_ReadyForUserInteraction); + } + } + // (through AccessBridge) private UIAElement abDescend(long hwnd, UIAElement parent, long vmid, long ac){ UIAElement modalElement = null; From 052988431edac8b385872fe32bf2cc40fd49ebbd Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 15 Oct 2021 09:47:00 +0200 Subject: [PATCH 31/40] Added ancestors debug information and improved HTML report by listing the type of text which could not be matched. --- .../HtmlReporting/HtmlSequenceReport.java | 8 ++++--- .../visualvalidation/VisualValidator.java | 10 ++++++--- .../extractor/ExpectedTextExtractorBase.java | 22 ++++++++++--------- .../ExpectedTextExtractorWebdriver.java | 4 ++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java b/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java index 68e0e763a..b1de01397 100644 --- a/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java +++ b/testar/src/nl/ou/testar/HtmlReporting/HtmlSequenceReport.java @@ -32,6 +32,7 @@ package nl.ou.testar.HtmlReporting; import nl.ou.testar.a11y.reporting.HTMLReporter; +import nl.ou.testar.visualvalidation.extractor.ExpectedElement; import nl.ou.testar.visualvalidation.matcher.CharacterMatchEntry; import nl.ou.testar.visualvalidation.matcher.ContentMatchResult; import nl.ou.testar.visualvalidation.matcher.LocationMatch; @@ -49,7 +50,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import java.util.stream.Collectors; public class HtmlSequenceReport implements Reporting { @@ -342,8 +342,10 @@ public void addVisualValidationResult(MatcherResult result, State state, @Nullab write("

    "); // List the expected elements without a location match. - result.getNoLocationMatches().forEach(it -> - write("

    No match for: \"" + it._text + "\" at location: " + it._location + "

    ") + result.getNoLocationMatches().forEach(it -> { + String type = it instanceof ExpectedElement ? "expected" : "ocr"; + write("

    No match for " + type + ": \"" + it._text + "\" at location: " + it._location + "

    "); + } ); // List the expected elements with a location match. diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index f644be012..70d400a03 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -185,9 +185,13 @@ private void processMatchResult(State state, AWTCanvas screenshot) { ); _matcherResult.getNoLocationMatches().forEach(result -> { - drawRectangle(copy, result._location, java.awt.Color.red); - // Overwrite the verdict if we couldn't find a location match for expected text. - validationVerdict[0] = createFailedToMatchVerdict(result); + if (result instanceof ExpectedElement) { + drawRectangle(copy, result._location, java.awt.Color.red); + // Overwrite the verdict if we couldn't find a location match for expected text. + validationVerdict[0] = createFailedToMatchVerdict(result); + } else { + drawRectangle(copy, result._location, Color.magenta); + } } ); } diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java index 022ff593c..d82256cbb 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -154,8 +154,12 @@ private void extractText() { private void extractedText(Rectangle applicationPosition, double displayScale, List expectedElements, Widget widget) { String widgetRole = widget.get(Role).name(); String text = widget.get(getVisualTextTag(widgetRole), ""); + StringBuilder sb = new StringBuilder(); + Util.ancestors(widget).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); + String ancestors = sb.toString(); + boolean ignored = true; - if (widgetIsIncluded(widget, widgetRole)) { + if (widgetIsIncluded(widget, widgetRole, ancestors)) { if (text != null && !text.isEmpty()) { Rectangle absoluteLocation = getLocation(widget); Location relativeLocation = new Location( @@ -164,26 +168,24 @@ private void extractedText(Rectangle applicationPosition, double displayScale, L (int) (absoluteLocation.width * displayScale), (int) (absoluteLocation.height * displayScale)); expectedElements.add(new ExpectedElement(relativeLocation, text)); - } - } else { - if (_loggingEnabled) { - Logger.log(Level.DEBUG, TAG, "Widget {} with role {} is ignored", text, widgetRole); + ignored = false; } } + + if (_loggingEnabled) { + Logger.log(Level.INFO, TAG, "Widget {} with role {} and ancestors {} is {}", + text, widgetRole, ancestors, ignored ? "ignored" : "added"); + } } - protected boolean widgetIsIncluded(Widget widget, String role) { + protected boolean widgetIsIncluded(Widget widget, String role, String ancestors) { boolean containsReadableText = true; if (_blacklist.containsKey(role)) { containsReadableText = false; try { List blacklistedAncestors = _blacklist.get(role); if (!blacklistedAncestors.isEmpty()) { - StringBuilder sb = new StringBuilder(); - Util.ancestors(widget).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); - // Check if we should ignore this widget based on its ancestors. - String ancestors = sb.toString(); containsReadableText = blacklistedAncestors.stream().noneMatch(ancestors::equals); } } catch (NullPointerException ignored) { diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java index 95f330f19..99a681d30 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java @@ -12,8 +12,8 @@ class ExpectedTextExtractorWebdriver extends ExpectedTextExtractorBase { } @Override - protected boolean widgetIsIncluded(Widget widget, String role) { + protected boolean widgetIsIncluded(Widget widget, String role, String ancestors) { // Check if the widget is visible - return widget.get(WebIsFullOnScreen) && super.widgetIsIncluded(widget, role); + return widget.get(WebIsFullOnScreen) && super.widgetIsIncluded(widget, role, ancestors); } } From 397e7bd9d0cb898315f7efd3a9af5abba736ebb1 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 15 Oct 2021 15:34:37 +0200 Subject: [PATCH 32/40] Fixed extracting application name when SUT takes '\' arguments. --- testar/src/org/testar/OutputStructure.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testar/src/org/testar/OutputStructure.java b/testar/src/org/testar/OutputStructure.java index 0379748be..c7b12c75e 100644 --- a/testar/src/org/testar/OutputStructure.java +++ b/testar/src/org/testar/OutputStructure.java @@ -97,9 +97,10 @@ public static void createOutputSUTname(Settings settings) { executedSUTname = domain; } else if (sutConnectorValue.contains(".exe")) { - int startSUT = sutConnectorValue.lastIndexOf(File.separator)+1; int endSUT = sutConnectorValue.indexOf(".exe"); - String sutName = sutConnectorValue.substring(startSUT, endSUT); + String pathWithoutArguments = sutConnectorValue.substring(0, endSUT); + int startSUT = pathWithoutArguments.lastIndexOf(File.separator)+1; + String sutName = pathWithoutArguments.substring(startSUT, endSUT); executedSUTname = sutName; } else if (sutConnectorValue.contains(".jar")) { From 6874a9ccb51fd4bdd33a2e4365b65ed3d82a1961 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Fri, 15 Oct 2021 15:35:39 +0200 Subject: [PATCH 33/40] Fixed screenshot capture form MS word. Due to visual border effects the old implementation selected the wrong element to determine the window size. --- core/src/es/upv/staq/testar/ProtocolUtil.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/src/es/upv/staq/testar/ProtocolUtil.java b/core/src/es/upv/staq/testar/ProtocolUtil.java index 6184eae0c..36a7417ce 100644 --- a/core/src/es/upv/staq/testar/ProtocolUtil.java +++ b/core/src/es/upv/staq/testar/ProtocolUtil.java @@ -245,10 +245,20 @@ public static String getStateshot(State state){ */ public static AWTCanvas getStateshotBinary(State state) { Shape viewPort = null; - if (state.childCount() > 0){ - viewPort = state.child(0).get(Tags.Shape, null); - if (viewPort != null && (viewPort.width() * viewPort.height() < 1)) - viewPort = null; + for (int index = 0; index < state.childCount(); index++){ + // While testing Word (2109 build 14430.20270) we noticed that the height of the screenshots was only 5px. + // After investigation, we noticed that root contained 5 children. Four had "MSO_BORDEREFFECT_WINDOW_CLASS" + // and had none children. And only one was called "OpusApp" and contained child elements. + // Ideally this check is more strict; (frameworkId == Win32 and classname != MSO_BORDEREFFECT_WINDOW_CLASS) + // but unfortunately these tags are not available at this generic level. Previous implementation directly + // used the first child. However, it makes more sense to select the widget which contains children. + if (state.child(index).childCount() != 0) { + viewPort = state.child(index).get(Tags.Shape, null); + if (viewPort != null && (viewPort.width() * viewPort.height() < 1)) { + viewPort = null; + } + break; + } } //If the state Shape is not properly obtained, or the State has an error, use full monitor screen From d59d7be19d34bcee50fe44c3b22d3d4e53dbb5a4 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Sat, 27 Nov 2021 23:00:10 +0100 Subject: [PATCH 34/40] Fixed replay for OCR recording. --- .../extractor/ExpectedTextExtractorBase.java | 23 +++++---- .../src/org/fruit/monkey/DefaultProtocol.java | 47 ++++++++++--------- .../testar/settings/ExtendedSettingFile.java | 2 +- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java index d82256cbb..a91dc0abc 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -59,16 +59,19 @@ public class ExpectedTextExtractorBase extends Thread implements TextExtractorIn ExpectedTextExtractorBase(Tag defaultTag) { WidgetTextConfiguration config = ExtendedSettingsFactory.createWidgetTextConfiguration(); // Load the extractor configuration into a lookup table for quick access. - config.widget.forEach(it -> { - if (it.ignore) { - List ancestor = it.ancestor.isEmpty() ? - Collections.emptyList() : Collections.singletonList(it.ancestor); - _blacklist.merge(it.role, ancestor, (list1, list2) -> - Stream.concat(list1.stream(), list2.stream()).collect(Collectors.toList())); - } else { - _lookupTable.put(it.role, it.tag); - } - }); + try { + config.widget.forEach(it -> { + if (it.ignore) { + List ancestor = it.ancestor.isEmpty() ? + Collections.emptyList() : Collections.singletonList(it.ancestor); + _blacklist.merge(it.role, ancestor, (list1, list2) -> + Stream.concat(list1.stream(), list2.stream()).collect(Collectors.toList())); + } else { + _lookupTable.put(it.role, it.tag); + } + }); + } catch (NullPointerException ignored) { + } _loggingEnabled = config.loggingEnabled; diff --git a/testar/src/org/fruit/monkey/DefaultProtocol.java b/testar/src/org/fruit/monkey/DefaultProtocol.java index 04f349cc6..9e882e8b1 100644 --- a/testar/src/org/fruit/monkey/DefaultProtocol.java +++ b/testar/src/org/fruit/monkey/DefaultProtocol.java @@ -1222,7 +1222,9 @@ protected void runReplayLoop(){ double rrt = settings.get(ConfigTags.ReplayRetryTime); - while(success && !faultySequence && mode() == Modes.Replay){ + boolean suppressFaultySequence = ExtendedSettingsFactory.createVisualValidationSettings().enabled; + + while(success && (suppressFaultySequence || !faultySequence) && mode() == Modes.Replay){ //Initialize local fragment and read saved action of PathToReplaySequence File Taggable replayableFragment; @@ -1741,19 +1743,7 @@ else if (actions.isEmpty()){ //TODO move the CPU metric to another helper class that is not default "TrashBinCode" or "SUTprofiler" //TODO check how well the CPU usage based waiting works protected boolean executeAction(SUT system, State state, Action action){ - AWTCanvas screenshot; - if (NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { - screenshot = WdProtocolUtil.getActionshot(state, action); - } else { - screenshot = ProtocolUtil.getActionshot(state, action); - } - state.set(ExecutedAction, action); - ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), screenshot); - - htmlReport.addVisualValidationResult( - visualValidationManager.AnalyzeImage(state, screenshot, action.get(OriginWidget, null)), - state, action - ); + takeActionScreenshot(state, action); double waitTime = settings.get(ConfigTags.TimeToWaitAfterAction); @@ -1779,16 +1769,11 @@ protected boolean executeAction(SUT system, State state, Action action){ return false; } } - + protected boolean replayAction(SUT system, State state, Action action, double actionWaitTime, double actionDuration){ - // Get an action screenshot based on the NativeLinker platform - if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { - WdProtocolUtil.getActionshot(state,action); - } else { - ProtocolUtil.getActionshot(state,action); - } + takeActionScreenshot(state, action); - try{ + try{ double halfWait = actionWaitTime == 0 ? 0.01 : actionWaitTime / 2.0; // seconds Util.pause(halfWait); // help for a better match of the state' actions visualization action.run(system, state, actionDuration); @@ -1811,6 +1796,24 @@ protected boolean replayAction(SUT system, State state, Action action, double ac } } + private void takeActionScreenshot(State state, Action action) { + AWTCanvas screenshot; + if (NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { + screenshot = WdProtocolUtil.getActionshot(state, action); + } else { + screenshot = ProtocolUtil.getActionshot(state, action); + } + state.set(ExecutedAction, action); + ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + screenshot); + + htmlReport.addVisualValidationResult( + visualValidationManager.AnalyzeImage(state, screenshot, action.get(OriginWidget, null)), + state, action + ); + } + /** * This method is here, so that ClickFilterLayerProtocol can override it, and the behaviour is updated * diff --git a/testar/src/org/testar/settings/ExtendedSettingFile.java b/testar/src/org/testar/settings/ExtendedSettingFile.java index b3b3dffda..8d780fd06 100644 --- a/testar/src/org/testar/settings/ExtendedSettingFile.java +++ b/testar/src/org/testar/settings/ExtendedSettingFile.java @@ -105,7 +105,7 @@ public class ExtendedSettingFile implements Serializable { /** * Constructor, each specialization must have a unique implementation of this class. * - * @param fileLocation The absolute path the the XML file. + * @param fileLocation The absolute path of the XML file. * @param fileAccessMutex Mutex for thread-safe access. */ protected ExtendedSettingFile(@NonNull String fileLocation, @NonNull ReentrantReadWriteLock fileAccessMutex) { From 35e981765be38b861968097ce62c2bfe75da1780 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Tue, 5 Apr 2022 21:53:18 +0200 Subject: [PATCH 35/40] Renamed VisualMatcher to VisualMatcherInterface. --- testar/src/nl/ou/testar/visualvalidation/VisualValidator.java | 4 ++-- .../visualvalidation/extractor/ExpectedTextExtractorBase.java | 2 +- .../ou/testar/visualvalidation/matcher/LocationMatcher.java | 2 +- .../testar/visualvalidation/matcher/VisualDummyMatcher.java | 2 +- .../testar/visualvalidation/matcher/VisualMatcherFactory.java | 4 ++-- .../{VisualMatcher.java => VisualMatcherInterface.java} | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename testar/src/nl/ou/testar/visualvalidation/matcher/{VisualMatcher.java => VisualMatcherInterface.java} (88%) diff --git a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java index 70d400a03..c519b4067 100644 --- a/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java +++ b/testar/src/nl/ou/testar/visualvalidation/VisualValidator.java @@ -7,7 +7,7 @@ import nl.ou.testar.visualvalidation.extractor.TextExtractorInterface; import nl.ou.testar.visualvalidation.matcher.ContentMatchResult; import nl.ou.testar.visualvalidation.matcher.MatcherResult; -import nl.ou.testar.visualvalidation.matcher.VisualMatcher; +import nl.ou.testar.visualvalidation.matcher.VisualMatcherInterface; import nl.ou.testar.visualvalidation.matcher.VisualMatcherFactory; import nl.ou.testar.visualvalidation.ocr.OcrConfiguration; import nl.ou.testar.visualvalidation.ocr.OcrEngineFactory; @@ -37,7 +37,7 @@ public class VisualValidator implements VisualValidationManager, OcrResultCallback, ExpectedTextCallback { static final Color darkOrange = new Color(255, 128, 0); private final String TAG = "VisualValidator"; - private final VisualMatcher _matcher; + private final VisualMatcherInterface _matcher; private final OcrEngineInterface _ocrEngine; private final Object _ocrResultSync = new Object(); private final AtomicBoolean _ocrResultReceived = new AtomicBoolean(); diff --git a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java index a91dc0abc..10c4505b4 100644 --- a/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java +++ b/testar/src/nl/ou/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -42,7 +42,7 @@ public class ExpectedTextExtractorBase extends Thread implements TextExtractorIn final private Tag defaultTag; private final boolean _loggingEnabled; /** - * A blacklist for {@link Widget}'s which should be ignored based on their {@code Role} because the don't contain + * A blacklist for {@link Widget}'s which should be ignored based on their {@code Role} because they don't contain * readable text. Optional when the value (which represents the ancestor path) is set, the {@link Widget} should * only be ignored when the ancestor path is equal with the {@link Widget} under investigation. */ diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java index 89ecb9258..59445fde2 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/LocationMatcher.java @@ -14,7 +14,7 @@ import java.util.TreeSet; import java.util.stream.Collectors; -public class LocationMatcher implements VisualMatcher { +public class LocationMatcher implements VisualMatcherInterface { static final String TAG = "Matcher"; private final MatcherConfiguration config; diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java index ee8da8efe..4836f8f19 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualDummyMatcher.java @@ -5,7 +5,7 @@ import java.util.List; -public class VisualDummyMatcher implements VisualMatcher { +public class VisualDummyMatcher implements VisualMatcherInterface { @Override public MatcherResult Match(List ocrResult, List expectedText) { return new MatcherResult(); diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java index 61ec20aac..dc855f638 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherFactory.java @@ -1,11 +1,11 @@ package nl.ou.testar.visualvalidation.matcher; public class VisualMatcherFactory { - public static VisualMatcher createDummyMatcher() { + public static VisualMatcherInterface createDummyMatcher() { return new VisualDummyMatcher(); } - public static VisualMatcher createLocationMatcher(MatcherConfiguration setting) { + public static VisualMatcherInterface createLocationMatcher(MatcherConfiguration setting) { return new LocationMatcher(setting); } } diff --git a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherInterface.java similarity index 88% rename from testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java rename to testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherInterface.java index 811f958ca..deff4debb 100644 --- a/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcher.java +++ b/testar/src/nl/ou/testar/visualvalidation/matcher/VisualMatcherInterface.java @@ -5,7 +5,7 @@ import java.util.List; -public interface VisualMatcher { +public interface VisualMatcherInterface { MatcherResult Match(List ocrResult, List expectedText); void destroy(); From 1e49d4da25f9356e651e482467da77e223870e29 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Thu, 7 Apr 2022 12:20:22 +0200 Subject: [PATCH 36/40] Fixed incorrect merge conflict. --- testar/src/org/testar/monkey/DefaultProtocol.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testar/src/org/testar/monkey/DefaultProtocol.java b/testar/src/org/testar/monkey/DefaultProtocol.java index 6c73d10a8..638f64c5c 100644 --- a/testar/src/org/testar/monkey/DefaultProtocol.java +++ b/testar/src/org/testar/monkey/DefaultProtocol.java @@ -332,7 +332,7 @@ public final void run(final Settings settings) { + "
    Surely exists a residual process \"chromedriver.exe\" running." + "
    You can use Task Manager to finish it."; - popupMessage(msg); + chromeDriverMissing(msg); System.out.println(msg); System.out.println(e.getMessage()); }else { From d66fb4af5c3cdfe1d4bb222dc5b439a267290e8f Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Thu, 7 Apr 2022 14:54:05 +0200 Subject: [PATCH 37/40] Disabled the visual validation by default. --- .../org/testar/visualvalidation/VisualValidationSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testar/src/org/testar/visualvalidation/VisualValidationSettings.java b/testar/src/org/testar/visualvalidation/VisualValidationSettings.java index 8f752997e..45980ff72 100644 --- a/testar/src/org/testar/visualvalidation/VisualValidationSettings.java +++ b/testar/src/org/testar/visualvalidation/VisualValidationSettings.java @@ -49,7 +49,7 @@ public class VisualValidationSettings extends ExtendedSettingBase Date: Thu, 7 Apr 2022 15:27:36 +0200 Subject: [PATCH 38/40] Added default value for obtaining the handle. Without default value CI will fail. --- windows/src/org/testar/monkey/alayer/windows/StateFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java b/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java index e75e17b90..013b1741d 100644 --- a/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java +++ b/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java @@ -104,7 +104,7 @@ public UIAState call() throws Exception { UIAState root = createWidgetTree(uiaRoot); root.set(Tags.Role, Roles.Process); root.set(Tags.NotResponding, false); - root.set(Tags.HWND, uiaRoot.children.get(0).get(Tags.HWND)); + root.set(Tags.HWND, uiaRoot.children.get(0).get(Tags.HWND, null)); for (Widget w : root) w.set(Tags.Path,Util.indexString(w)); if (system != null && (root == null || root.childCount() == 0) && system.getNativeAutomationCache() != null) From 4c79ce4241f2dbfb6840a476e618dc519152c014 Mon Sep 17 00:00:00 2001 From: Tycho Menting Date: Thu, 7 Apr 2022 16:18:40 +0200 Subject: [PATCH 39/40] Set to 0 instead of null to bypass a null check. --- windows/src/org/testar/monkey/alayer/windows/StateFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java b/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java index 013b1741d..637b2e5fc 100644 --- a/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java +++ b/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java @@ -104,7 +104,7 @@ public UIAState call() throws Exception { UIAState root = createWidgetTree(uiaRoot); root.set(Tags.Role, Roles.Process); root.set(Tags.NotResponding, false); - root.set(Tags.HWND, uiaRoot.children.get(0).get(Tags.HWND, null)); + root.set(Tags.HWND, uiaRoot.children.get(0).get(Tags.HWND, 0L)); for (Widget w : root) w.set(Tags.Path,Util.indexString(w)); if (system != null && (root == null || root.childCount() == 0) && system.getNativeAutomationCache() != null) From 0e89fd80dee9ff82a55bbdb31c49fe9fc5ff518c Mon Sep 17 00:00:00 2001 From: Fernando Pastor <32359126+ferpasri@users.noreply.github.com> Date: Tue, 27 Sep 2022 20:06:00 +0200 Subject: [PATCH 40/40] update mockito dependency --- testar/build.gradle | 2 -- .../testar/visualvalidation/matcher/LocationMatcherTest.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/testar/build.gradle b/testar/build.gradle index 59347e87f..f73805624 100644 --- a/testar/build.gradle +++ b/testar/build.gradle @@ -135,8 +135,6 @@ dependencies { implementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.1' implementation group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.1' implementation group: 'com.orientechnologies', name: 'orientdb-graphdb', version: '3.0.34' - - testImplementation group: 'org.mockito', name: 'mockito-all', version: '1.10.19' } task prepareSettings(type: Copy) { diff --git a/testar/test/org/testar/visualvalidation/matcher/LocationMatcherTest.java b/testar/test/org/testar/visualvalidation/matcher/LocationMatcherTest.java index 581082ff8..a1305e79f 100644 --- a/testar/test/org/testar/visualvalidation/matcher/LocationMatcherTest.java +++ b/testar/test/org/testar/visualvalidation/matcher/LocationMatcherTest.java @@ -4,7 +4,7 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.testar.visualvalidation.Location; import org.testar.visualvalidation.extractor.ExpectedElement; import org.testar.visualvalidation.ocr.RecognizedElement;
    \"" + contentResult.expectedText + "\" " + - "[" + contentResult.totalMatched + "/" + contentResult.totalExpected + "]