From 7a3fbbc3bf4820024a812fe6df83008c60786fe1 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Fri, 31 Jan 2020 03:51:31 -0500 Subject: [PATCH 1/6] Better handling of page scrolling to elements --- seleniumbase/fixtures/base_case.py | 22 +++++++++++++--------- seleniumbase/fixtures/js_utils.py | 9 +++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 184dd5acd78..6c602071e35 100755 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -133,7 +133,7 @@ def click(self, selector, by=By.CSS_SELECTOR, timeout=None, delay=0): self.driver, selector, by, timeout=timeout) self.__demo_mode_highlight_if_active(selector, by) if not self.demo_mode: - self.__scroll_to_element(element) + self.__scroll_to_element(element, selector, by) pre_action_url = self.driver.current_url if delay and delay > 0: time.sleep(delay) @@ -208,7 +208,7 @@ def double_click(self, selector, by=By.CSS_SELECTOR, timeout=None): self.driver, selector, by, timeout=timeout) self.__demo_mode_highlight_if_active(selector, by) if not self.demo_mode: - self.__scroll_to_element(element) + self.__scroll_to_element(element, selector, by) pre_action_url = self.driver.current_url try: actions = ActionChains(self.driver) @@ -297,7 +297,7 @@ def update_text(self, selector, new_value, by=By.CSS_SELECTOR, selector, by=by, timeout=timeout) self.__demo_mode_highlight_if_active(selector, by) if not self.demo_mode: - self.__scroll_to_element(element) + self.__scroll_to_element(element, selector, by) try: element.clear() except (StaleElementReferenceException, ENI_Exception): @@ -368,7 +368,7 @@ def add_text(self, selector, text, by=By.CSS_SELECTOR, timeout=None): selector, by=by, timeout=timeout) self.__demo_mode_highlight_if_active(selector, by) if not self.demo_mode: - self.__scroll_to_element(element) + self.__scroll_to_element(element, selector, by) pre_action_url = self.driver.current_url try: if not text.endswith('\n'): @@ -1797,13 +1797,13 @@ def scroll_to(self, selector, by=By.CSS_SELECTOR, timeout=None): element = self.wait_for_element_visible( selector, by=by, timeout=timeout) try: - self.__scroll_to_element(element) + self.__scroll_to_element(element, selector, by) except (StaleElementReferenceException, ENI_Exception): self.wait_for_ready_state_complete() time.sleep(0.05) element = self.wait_for_element_visible( selector, by=by, timeout=timeout) - self.__scroll_to_element(element) + self.__scroll_to_element(element, selector, by) def slow_scroll_to(self, selector, by=By.CSS_SELECTOR, timeout=None): ''' Slow motion scroll to destination ''' @@ -1846,7 +1846,7 @@ def js_click(self, selector, by=By.CSS_SELECTOR, all_matches=False): if self.is_element_visible(selector, by=by): self.__demo_mode_highlight_if_active(selector, by) if not self.demo_mode: - self.__scroll_to_element(element) + self.__scroll_to_element(element, selector, by) css_selector = self.convert_to_css_selector(selector, by=by) css_selector = re.escape(css_selector) css_selector = self.__escape_quotes_if_needed(css_selector) @@ -4047,8 +4047,12 @@ def __demo_mode_highlight_if_active(self, selector, by): selector, by=by, timeout=settings.SMALL_TIMEOUT) self.__slow_scroll_to_element(element) - def __scroll_to_element(self, element): - js_utils.scroll_to_element(self.driver, element) + def __scroll_to_element(self, element, selector=None, by=By.CSS_SELECTOR): + success = js_utils.scroll_to_element(self.driver, element) + if not success and selector: + self.wait_for_ready_state_complete() + element = page_actions.wait_for_element_visible( + self.driver, selector, by, timeout=settings.SMALL_TIMEOUT) self.__demo_mode_pause_if_active(tiny=True) def __slow_scroll_to_element(self, element): diff --git a/seleniumbase/fixtures/js_utils.py b/seleniumbase/fixtures/js_utils.py index cd610211c2b..c4ec54e8c38 100755 --- a/seleniumbase/fixtures/js_utils.py +++ b/seleniumbase/fixtures/js_utils.py @@ -640,8 +640,8 @@ def scroll_to_element(driver, element): try: element_location = element.location['y'] except Exception: - element.location_once_scrolled_into_view - return + # element.location_once_scrolled_into_view # Old hack + return False element_location = element_location - 130 if element_location < 0: element_location = 0 @@ -650,8 +650,9 @@ def scroll_to_element(driver, element): # scroll_script = "jQuery('%s')[0].scrollIntoView()" % selector try: driver.execute_script(scroll_script) - except WebDriverException: - pass # Older versions of Firefox experienced issues here + return True + except Exception: + return False def slow_scroll_to_element(driver, element, browser): From d247608e2a3242b2ad76ca59253aa988880c41d2 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Fri, 31 Jan 2020 03:52:24 -0500 Subject: [PATCH 2/6] Better exception handling around log folders --- seleniumbase/core/log_helper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/seleniumbase/core/log_helper.py b/seleniumbase/core/log_helper.py index a0ad876a186..28f169a149d 100755 --- a/seleniumbase/core/log_helper.py +++ b/seleniumbase/core/log_helper.py @@ -168,8 +168,11 @@ def log_folder_setup(log_path, archive_logs=False): archived_folder, int(time.time())) if len(os.listdir(log_path)) > 0: - shutil.move(log_path, archived_logs) - os.makedirs(log_path) + try: + shutil.move(log_path, archived_logs) + os.makedirs(log_path) + except Exception: + pass # A file was probably open at the time if not settings.ARCHIVE_EXISTING_LOGS and not archive_logs: shutil.rmtree(archived_logs) else: From 415a57d3bb3afd726f72cd4362bdf4e05c28d04d Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Fri, 31 Jan 2020 04:14:47 -0500 Subject: [PATCH 3/6] Improve automated visual testing --- seleniumbase/fixtures/base_case.py | 59 +++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 6c602071e35..45b938ff286 100755 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -3482,6 +3482,36 @@ def wait_for_and_switch_to_alert(self, timeout=None): ############ + def __assert_eq(self, *args, **kwargs): + """ Minified assert_equal() using only the list diff. """ + minified_exception = None + try: + self.assertEqual(*args, **kwargs) + except Exception as e: + str_e = str(e) + minified_exception = "\nAssertionError:\n" + lines = str_e.split('\n') + countdown = 3 + countdown_on = False + for line in lines: + if countdown_on: + minified_exception += line + '\n' + countdown = countdown - 1 + if countdown == 0: + countdown_on = False + elif line.startswith('F'): + countdown_on = True + countdown = 3 + minified_exception += line + '\n' + elif line.startswith('+') or line.startswith('-'): + minified_exception += line + '\n' + elif line.startswith('?'): + minified_exception += line + '\n' + elif line.strip().startswith('*'): + minified_exception += line + '\n' + if minified_exception: + raise Exception(minified_exception) + def check_window(self, name="default", level=0, baseline=False): """ *** Automated Visual Testing with SeleniumBase *** @@ -3644,40 +3674,41 @@ def check_window(self, name="default", level=0, baseline=False): f.close() domain_fail = ( - "Page Domain Mismatch Failure: " + "\nPage Domain Mismatch Failure: " "Current Page Domain doesn't match the Page Domain of the " "Baseline! Can't compare two completely different sites! " "Run with --visual_baseline to reset the baseline!") level_1_failure = ( - "\n\n*** Exception: Visual Diff Failure:\n" + "\n*\n*** Exception: Visual Diff Failure:\n" "* HTML tags don't match the baseline!") level_2_failure = ( - "\n\n*** Exception: Visual Diff Failure:\n" - "* HTML tag attributes don't match the baseline!") + "\n*\n*** Exception: Visual Diff Failure:\n" + "* HTML tag attribute names don't match the baseline!") level_3_failure = ( - "\n\n*** Exception: Visual Diff Failure:\n" + "\n*\n*** Exception: Visual Diff Failure:\n" "* HTML tag attribute values don't match the baseline!") page_domain = self.get_domain_url(page_url) page_data_domain = self.get_domain_url(page_url_data) unittest.TestCase.maxDiff = 1000 - if level == 1 or level == 2 or level == 3: - self.assertEqual(page_domain, page_data_domain, domain_fail) - self.assertEqual(level_1, level_1_data, level_1_failure) + if level != 0: + self.assertEqual(page_data_domain, page_domain, domain_fail) unittest.TestCase.maxDiff = None - if level == 2 or level == 3: - self.assertEqual(level_2, level_2_data, level_2_failure) if level == 3: - self.assertEqual(level_3, level_3_data, level_3_failure) + self.__assert_eq(level_3_data, level_3, level_3_failure) + if level == 2: + self.__assert_eq(level_2_data, level_2, level_2_failure) + unittest.TestCase.maxDiff = 1000 + if level == 1: + self.__assert_eq(level_1_data, level_1, level_1_failure) + unittest.TestCase.maxDiff = None if level == 0: try: unittest.TestCase.maxDiff = 1000 self.assertEqual( page_domain, page_data_domain, domain_fail) - self.assertEqual(level_1, level_1_data, level_1_failure) unittest.TestCase.maxDiff = None - self.assertEqual(level_2, level_2_data, level_2_failure) - self.assertEqual(level_3, level_3_data, level_3_failure) + self.__assert_eq(level_3_data, level_3, level_3_failure) except Exception as e: print(e) # Level-0 Dry Run (Only print the differences) From 648b65b25210c6a1c9e177781bcd689a68bc6cfd Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Fri, 31 Jan 2020 04:17:14 -0500 Subject: [PATCH 4/6] Add examples for automated visual testing --- examples/visual_testing/ReadMe.md | 76 +++++++++++++++++---- examples/visual_testing/layout_test.py | 8 +-- examples/visual_testing/python_home_test.py | 12 ++++ examples/visual_testing/test_layout_fail.py | 30 ++++++++ examples/visual_testing/xkcd_visual_test.py | 13 ++++ 5 files changed, 122 insertions(+), 17 deletions(-) create mode 100755 examples/visual_testing/python_home_test.py create mode 100755 examples/visual_testing/test_layout_fail.py create mode 100755 examples/visual_testing/xkcd_visual_test.py diff --git a/examples/visual_testing/ReadMe.md b/examples/visual_testing/ReadMe.md index 1b8e722f401..45a495e2dfa 100755 --- a/examples/visual_testing/ReadMe.md +++ b/examples/visual_testing/ReadMe.md @@ -1,7 +1,7 @@ [](https://github.com/seleniumbase/SeleniumBase/blob/master/README.md) ### Automated Visual Testing (Layout Change Detection) -Automated visual testing can help you detect when something has changed the layout of a web page. Rather than comparing screenshots, a more effective way is to detect layout differences by comparing HTML tags and properties. If a change is detected, it could mean that something broke on the webpage, or possibly something harmless like a website redesign or dynamic content. +Automated visual testing helps you detect when the layout of a web page has changed. Rather than comparing screenshots, layout differences are detected by comparing HTML tags and properties with a baseline. If a change is detected, it could mean that something broke, the web page was redesigned, or dynamic content changed. To handle automated visual testing, SeleniumBase uses the ``self.check_window()`` method, which can set visual baselines for comparison and then compare the latest versions of web pages to the existing baseline. @@ -16,7 +16,7 @@ After the first time ``self.check_window()`` is called, later calls will compare Here's an example call: ``` -self.check_window(name="first_test)", level=1) +self.check_window(name="first_test)", level=3) ``` On the first run (or if the baseline is being set/reset) the "level" doesn't matter because that's only used for comparing the current layout to the existing baseline. @@ -26,12 +26,9 @@ Here's how the level system works: * level=1 -> HTML tags are compared to tags_level1.txt * level=2 -> - HTML tags are compared to tags_level1.txt and - HTML tags/attributes are compared to tags_level2.txt + HTML tags and attribute names are compared to tags_level2.txt * level=3 -> - HTML tags are compared to tags_level1.txt and - HTML tags + attributes are compared to tags_level2.txt and - HTML tags + attributes/values are compared to tags_level3.txt + HTML tags and attribute values are compared to tags_level3.txt As shown, Level-3 is the most strict, Level-1 is the least strict. If the comparisons from the latest window to the existing baseline don't match, the current test will fail, except for Level-0 tests. @@ -44,7 +41,7 @@ As long as ``--visual_baseline`` is used on the command line while running tests If you want to use ``self.check_window()`` to compare a web page to a later version of itself from within the same test run, you can add the parameter ``baseline=True`` to the first time you call ``self.check_window()`` in a test to use that as the baseline. This only makes sense if you're calling ``self.check_window()`` more than once with the same "name" parameter in the same test. -Automated Visual Testing with ``self.check_window()`` is not very effective for websites that have dynamic content because that changes the layout and structure of web pages. For those pages, you're much better off using regular SeleniumBase functional testing. +Automated Visual Testing with ``self.check_window()`` is not very effective for websites that have dynamic content because that changes the layout and structure of web pages. For those pages, you're much better off using regular SeleniumBase functional testing, unless you can remove the dynamic content before performing the comparison, (such as by using ``self.ad_block()`` to remove dynamic ad content on a web page). Example usage of ``self.check_window()``: ```python @@ -65,16 +62,16 @@ from seleniumbase import BaseCase class VisualLayoutTest(BaseCase): - def test_applitools_helloworld(self): + def test_applitools_layout_change(self): self.open('https://applitools.com/helloworld?diff1') - print('Creating baseline in "visual_baseline" folder...') + print('\nCreating baseline in "visual_baseline" folder...') self.check_window(name="helloworld", baseline=True) self.click('a[href="?diff1"]') # Verify html tags match previous version self.check_window(name="helloworld", level=1) - # Verify html tags + attributes match previous version + # Verify html tags and attribute names match previous version self.check_window(name="helloworld", level=2) - # Verify html tags + attributes + values match previous version + # Verify html tags and attribute values match previous version self.check_window(name="helloworld", level=3) # Change the page enough for a Level-3 comparison to fail self.click("button") @@ -82,7 +79,60 @@ class VisualLayoutTest(BaseCase): self.check_window(name="helloworld", level=2) with self.assertRaises(Exception): self.check_window(name="helloworld", level=3) - # Now that we know the exception was raised as expected, + # Now that we know the Exception was raised as expected, # let's print out the comparison results by running in Level-0. + # (NOTE: Running with level-0 will print but NOT raise an Exception.) self.check_window(name="helloworld", level=0) ``` + +Here's the output of that: +``` +AssertionError: +First differing element 39: +['div', [['class', ['section', 'hidden-section', 'image-section']]]] +['div', [['class', ['section', 'image-section']]]] + +- ['div', [['class', ['section', 'hidden-section', 'image-section']]]], +? ------------------ ++ ['div', [['class', ['section', 'image-section']]]], +* +*** Exception: Visual Diff Failure: +* HTML tag attribute values don't match the baseline! +``` + +Here's another example: +```python +from seleniumbase import BaseCase + + +class VisualLayoutTest(BaseCase): + + def test_xkcd_layout_change(self): + self.open('https://xkcd.com/554/') + print('\nCreating baseline in "visual_baseline" folder.') + self.check_window(name="xkcd_554", baseline=True) + # Change height: (83 -> 130) , Change width: (185 -> 120) + self.set_attribute('[alt="xkcd.com logo"]', "height", "130") + self.set_attribute('[alt="xkcd.com logo"]', "width", "120") + self.check_window(name="xkcd_554", level=0) +``` + +Here's the output of that: +``` +AssertionError: +First differing element 22: +['img[30 chars]['height', '83'], ['src', '/s/0b7742.png'], ['width', '185']]] +['img[30 chars]['height', '130'], ['src', '/s/0b7742.png'], ['width', '120']]] + +- ['height', '83'], +? ^ ++ ['height', '130'], +? ^ + +- ['width', '185']]], +? ^^ ++ ['width', '120']]], +? ^^ +* +*** Exception: Visual Diff Failure: +* HTML tag attribute values don't match the baseline! +``` diff --git a/examples/visual_testing/layout_test.py b/examples/visual_testing/layout_test.py index dd12253a264..1c4887e4072 100755 --- a/examples/visual_testing/layout_test.py +++ b/examples/visual_testing/layout_test.py @@ -3,16 +3,16 @@ class VisualLayoutTest(BaseCase): - def test_applitools_helloworld(self): + def test_applitools_layout_change(self): self.open('https://applitools.com/helloworld?diff1') - print('Creating baseline in "visual_baseline" folder...') + print('\nCreating baseline in "visual_baseline" folder...') self.check_window(name="helloworld", baseline=True) self.click('a[href="?diff1"]') # Verify html tags match previous version self.check_window(name="helloworld", level=1) - # Verify html tags + attributes match previous version + # Verify html tags and attribute names match previous version self.check_window(name="helloworld", level=2) - # Verify html tags + attributes + values match previous version + # Verify html tags and attribute values match previous version self.check_window(name="helloworld", level=3) # Change the page enough for a Level-3 comparison to fail self.click("button") diff --git a/examples/visual_testing/python_home_test.py b/examples/visual_testing/python_home_test.py new file mode 100755 index 00000000000..c8def7c6cc2 --- /dev/null +++ b/examples/visual_testing/python_home_test.py @@ -0,0 +1,12 @@ +from seleniumbase import BaseCase + + +class VisualLayoutTest(BaseCase): + + def test_python_home_layout_change(self): + self.open('https://python.org/') + print('\nCreating baseline in "visual_baseline" folder.') + self.check_window(name="github_home", baseline=True) + # Remove the "Donate" button + self.remove_element('a.donate-button') + self.check_window(name="github_home", level=0) diff --git a/examples/visual_testing/test_layout_fail.py b/examples/visual_testing/test_layout_fail.py new file mode 100755 index 00000000000..dd6a7d050a9 --- /dev/null +++ b/examples/visual_testing/test_layout_fail.py @@ -0,0 +1,30 @@ +from seleniumbase import BaseCase + + +class VisualLayoutFailureTest(BaseCase): + + def test_applitools_layout_change_failure(self): + self.open('https://applitools.com/helloworld?diff1') + print('\nCreating baseline in "visual_baseline" folder.') + self.check_window(name="helloworld", baseline=True) + self.click('a[href="?diff1"]') + # Change the page enough for a Level-3 comparison to fail + self.click("button") + self.check_window(name="helloworld", level=3) + + def test_python_home_layout_change_failure(self): + self.open('https://python.org/') + print('\nCreating baseline in "visual_baseline" folder.') + self.check_window(name="python_home", baseline=True) + # Remove the "Donate" button + self.remove_element('a.donate-button') + self.check_window(name="python_home", level=3) + + def test_xkcd_layout_change_failure(self): + self.open('https://xkcd.com/554/') + print('\nCreating baseline in "visual_baseline" folder.') + self.check_window(name="xkcd_554", baseline=True) + # Change height: (83 -> 110) , Change width: (185 -> 120) + self.set_attribute('[alt="xkcd.com logo"]', "height", "110") + self.set_attribute('[alt="xkcd.com logo"]', "width", "120") + self.check_window(name="xkcd_554", level=3) diff --git a/examples/visual_testing/xkcd_visual_test.py b/examples/visual_testing/xkcd_visual_test.py new file mode 100755 index 00000000000..3b619c14c35 --- /dev/null +++ b/examples/visual_testing/xkcd_visual_test.py @@ -0,0 +1,13 @@ +from seleniumbase import BaseCase + + +class VisualLayoutTest(BaseCase): + + def test_xkcd_layout_change(self): + self.open('https://xkcd.com/554/') + print('\nCreating baseline in "visual_baseline" folder.') + self.check_window(name="xkcd_554", baseline=True) + # Change height: (83 -> 130) , Change width: (185 -> 120) + self.set_attribute('[alt="xkcd.com logo"]', "height", "130") + self.set_attribute('[alt="xkcd.com logo"]', "width", "120") + self.check_window(name="xkcd_554", level=0) From fbea75401b1a6af2bf557763722201d1cc2c80f7 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Fri, 31 Jan 2020 04:18:16 -0500 Subject: [PATCH 5/6] Update pytest and wheel --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4ca9db90e66..bb38a172d8a 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ pip>=20.0.2 setuptools>=44.0.0;python_version<"3" setuptools>=45.1.0;python_version>="3" setuptools-scm>=3.4.3 -wheel>=0.34.1 +wheel>=0.34.2 six==1.14.0 nose==1.3.7 ipdb==0.12.3 @@ -14,7 +14,7 @@ selenium==3.141.0 pluggy>=0.13.1 attrs>=19.3.0 pytest>=4.6.9;python_version<"3" -pytest>=5.3.4;python_version>="3" +pytest>=5.3.5;python_version>="3" pytest-cov>=2.8.1 pytest-forked>=1.1.3 pytest-html==1.22.1;python_version<"3.6" diff --git a/setup.py b/setup.py index d17667e5ae6..64fb2319661 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'pluggy>=0.13.1', 'attrs>=19.3.0', 'pytest>=4.6.9;python_version<"3"', # For Python 2 compatibility - 'pytest>=5.3.4;python_version>="3"', + 'pytest>=5.3.5;python_version>="3"', 'pytest-cov>=2.8.1', 'pytest-forked>=1.1.3', 'pytest-html==1.22.1;python_version<"3.6"', From d6e7edfcba0a175578c3a1c63754290c242b4f59 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Fri, 31 Jan 2020 04:20:28 -0500 Subject: [PATCH 6/6] Version 1.35.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 64fb2319661..3a7d7b06e43 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup( name='seleniumbase', - version='1.34.28', + version='1.35.0', description='Fast, Easy, and Reliable Browser Automation & Testing.', long_description=long_description, long_description_content_type='text/markdown',