diff --git a/.gitignore b/.gitignore index 0a63fe763..d30dfda65 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ dist .cache __pycache__ .idea +.pytest_cache diff --git a/appium/webdriver/common/mobileby.py b/appium/webdriver/common/mobileby.py index be0b68eb9..3e77c51eb 100644 --- a/appium/webdriver/common/mobileby.py +++ b/appium/webdriver/common/mobileby.py @@ -21,3 +21,4 @@ class MobileBy(By): IOS_CLASS_CHAIN = '-ios class chain' ANDROID_UIAUTOMATOR = '-android uiautomator' ACCESSIBILITY_ID = 'accessibility id' + IMAGE = '-image' diff --git a/appium/webdriver/imagelement.py b/appium/webdriver/imagelement.py new file mode 100644 index 000000000..252599e1d --- /dev/null +++ b/appium/webdriver/imagelement.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + + +class ImageElement: + + def __init__(self, driver, x, y, width, height): + self.driver = driver + self.center_x = math.floor(x + width / 2) + self.center_y = math.floor(y + height / 2) + self.x = x + self.y = y + self.width = width + self.height = height + + def click(self): + """ + Clicks in the middle of an image bounds + """ + return self.driver.tap([(self.center_x, self.center_y)]) + + @property + def size(self): + return {'width': self.width, 'height': self.height} + + @property + def location(self): + return {'x': self.x, 'y': self.y} + + @property + def rect(self): + return { + 'width': self.width, + 'height': self.height, + 'x': self.x, + 'y': self.y + } + + def is_displayed(self): + return True diff --git a/appium/webdriver/webdriver.py b/appium/webdriver/webdriver.py index 577674f5a..291301a36 100644 --- a/appium/webdriver/webdriver.py +++ b/appium/webdriver/webdriver.py @@ -18,6 +18,7 @@ from .errorhandler import MobileErrorHandler from .switch_to import MobileSwitchTo from .webelement import WebElement as MobileWebElement +from .imagelement import ImageElement from appium.webdriver.clipboard_content_type import ClipboardContentType from appium.webdriver.common.mobileby import MobileBy @@ -26,13 +27,16 @@ from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait -from selenium.common.exceptions import TimeoutException, InvalidArgumentException +from selenium.common.exceptions import (TimeoutException, + WebDriverException, InvalidArgumentException, NoSuchElementException) from selenium.webdriver.remote.command import Command as RemoteCommand import base64 import copy +DEFAULT_MATCH_THRESHOLD = 0.5 + # From remote/webdriver.py _W3C_CAPABILITY_NAMES = frozenset([ 'acceptInsecureCerts', @@ -84,7 +88,9 @@ def _make_w3c_caps(caps): always_match['moz:firefoxOptions'] = new_opts return {'firstMatch': [{}], 'alwaysMatch': always_match} + class WebDriver(webdriver.Remote): + def __init__(self, command_executor='http://127.0.0.1:4444/wd/hub', desired_capabilities=None, browser_profile=None, proxy=None, keep_alive=False): @@ -102,6 +108,7 @@ def __init__(self, command_executor='http://127.0.0.1:4444/wd/hub', By.IOS_CLASS_CHAIN = MobileBy.IOS_CLASS_CHAIN By.ANDROID_UIAUTOMATOR = MobileBy.ANDROID_UIAUTOMATOR By.ACCESSIBILITY_ID = MobileBy.ACCESSIBILITY_ID + By.IMAGE = MobileBy.IMAGE def start_session(self, capabilities, browser_profile=None): """ @@ -206,6 +213,9 @@ def find_element(self, by=By.ID, value=None): # elif by == By.NAME: # by = By.CSS_SELECTOR # value = '[name="%s"]' % value + if by == By.IMAGE: + return self.find_element_by_image(value) + return self.execute(RemoteCommand.FIND_ELEMENT, { 'using': by, 'value': value})['value'] @@ -235,6 +245,10 @@ def find_elements(self, by=By.ID, value=None): # Return empty list if driver returns null # See https://github.com/SeleniumHQ/selenium/issues/4555 + + if by == By.IMAGE: + return self.find_elements_by_image(value) + return self.execute(RemoteCommand.FIND_ELEMENTS, { 'using': by, 'value': value})['value'] or [] @@ -327,6 +341,52 @@ def find_elements_by_android_uiautomator(self, uia_string): """ return self.find_elements(by=By.ANDROID_UIAUTOMATOR, value=uia_string) + def find_element_by_image(self, png_img_path, + match_threshold=DEFAULT_MATCH_THRESHOLD): + """Finds a portion of a screenshot by an image. + Uses driver.find_image_occurrence under the hood. + + :Args: + - png_img_path - a string corresponding to the path of a PNG image + - match_threshold - a double between 0 and 1 below which matches will + be rejected as element not found + + :return: an ImageElement object + """ + screenshot = self.get_screenshot_as_base64() + with open(png_img_path, 'rb') as png_file: + b64_data = base64.encodestring(png_file.read()) + try: + res = self.find_image_occurrence(screenshot, b64_data, + threshold=match_threshold) + except WebDriverException as e: + if 'Cannot find any occurrences' in str(e): + raise NoSuchElementException(e) + raise + rect = res['rect'] + return ImageElement(self, rect['x'], rect['y'], rect['width'], + rect['height']) + + def find_elements_by_image(self, png_img_path, + match_threshold=DEFAULT_MATCH_THRESHOLD): + """Finds a portion of a screenshot by an image. + Uses driver.find_image_occurrence under the hood. Note that this will + only ever return at most one element + + :Args: + - png_img_path - a string corresponding to the path of a PNG image + - match_threshold - a double between 0 and 1 below which matches will + be rejected as element not found + + :return: possibly-empty list of ImageElements + """ + els = [] + try: + els.append(self.find_element_by_image(png_img_path, match_threshold)) + except NoSuchElementException: + pass + return els + def find_element_by_accessibility_id(self, id): """Finds an element by accessibility id. @@ -1419,4 +1479,4 @@ def _addCommands(self): self.command_executor._commands[Command.GET_CLIPBOARD] = \ ('POST', '/session/$sessionId/appium/device/get_clipboard') self.command_executor._commands[Command.COMPARE_IMAGES] = \ - ('POST', '/session/$sessionId/appium/compare_images') \ No newline at end of file + ('POST', '/session/$sessionId/appium/compare_images') diff --git a/test/functional/android/desired_capabilities.py b/test/functional/android/desired_capabilities.py index b96bfbb08..84cb594fd 100644 --- a/test/functional/android/desired_capabilities.py +++ b/test/functional/android/desired_capabilities.py @@ -14,16 +14,17 @@ import os + # Returns abs path relative to this file and not cwd -PATH = lambda p: os.path.abspath( - os.path.join(os.path.dirname(__file__), p) -) +def PATH(p): + return os.path.abspath( + os.path.join(os.path.dirname(__file__), p) + ) def get_desired_capabilities(app): desired_caps = { 'platformName': 'Android', - 'platformVersion': '4.2', 'deviceName': 'Android Emulator', 'app': PATH('../../apps/' + app), 'newCommandTimeout': 240 diff --git a/test/functional/android/find_by_image_failure.png b/test/functional/android/find_by_image_failure.png new file mode 100644 index 000000000..5ea155cc4 Binary files /dev/null and b/test/functional/android/find_by_image_failure.png differ diff --git a/test/functional/android/find_by_image_success.png b/test/functional/android/find_by_image_success.png new file mode 100644 index 000000000..d7046ba8d Binary files /dev/null and b/test/functional/android/find_by_image_success.png differ diff --git a/test/functional/android/find_by_image_tests.py b/test/functional/android/find_by_image_tests.py new file mode 100644 index 000000000..9885843b4 --- /dev/null +++ b/test/functional/android/find_by_image_tests.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from appium import webdriver +from selenium.common.exceptions import NoSuchElementException, TimeoutException +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +import desired_capabilities + + +class FindByImageTests(unittest.TestCase): + + def setUp(self): + desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') + self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) + + def tearDown(self): + self.driver.quit() + + def test_find_based_on_image_template(self): + image_path = desired_capabilities.PATH('find_by_image_success.png') + el = WebDriverWait(self.driver, 3).until( + EC.presence_of_element_located((By.IMAGE, image_path)) + ) + size = el.size + self.assertIsNotNone(size['width']) + self.assertIsNotNone(size['height']) + loc = el.location + self.assertIsNotNone(loc['x']) + self.assertIsNotNone(loc['y']) + rect = el.rect + self.assertIsNotNone(rect['width']) + self.assertIsNotNone(rect['height']) + self.assertIsNotNone(rect['x']) + self.assertIsNotNone(rect['y']) + self.assertTrue(el.is_displayed()) + el.click() + self.driver.find_element_by_accessibility_id("Alarm") + + def test_find_multiple_elements_by_image_just_returns_one(self): + WebDriverWait(self.driver, 3).until( + EC.presence_of_element_located((By.ACCESSIBILITY_ID, "App")) + ) + image_path = desired_capabilities.PATH('find_by_image_success.png') + els = self.driver.find_elements_by_image(image_path) + els[0].click() + self.driver.find_element_by_accessibility_id("Alarm") + + def test_find_throws_no_such_element(self): + image_path = desired_capabilities.PATH('find_by_image_failure.png') + with self.assertRaises(TimeoutException): + WebDriverWait(self.driver, 3).until( + EC.presence_of_element_located((By.IMAGE, image_path)) + ) + with self.assertRaises(NoSuchElementException): + self.driver.find_element_by_image(image_path) + + +if __name__ == "__main__": + suite = unittest.TestLoader().loadTestsFromTestCase(FindByImageTests) + unittest.TextTestRunner(verbosity=2).run(suite)