From be56c982c0805f62a2948d9d7a3e21215c352174 Mon Sep 17 00:00:00 2001 From: Sarah Boyce <42296566+sarahboyce@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:01:58 +0200 Subject: [PATCH] Refs #34043 -- Added --screenshots option to runtests.py and selenium tests. --- .gitignore | 1 + django/test/selenium.py | 65 ++++++++++++++++++- .../contributing/writing-code/unit-tests.txt | 31 +++++++++ docs/releases/5.1.txt | 3 + tests/admin_views/tests.py | 15 ++++- tests/runtests.py | 14 +++- 6 files changed, 126 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 6a67c6eb28..7b065ff5fc 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ tests/coverage_html/ tests/.coverage* build/ tests/report/ +tests/screenshots/ diff --git a/django/test/selenium.py b/django/test/selenium.py index ff83299b19..07df34ae1f 100644 --- a/django/test/selenium.py +++ b/django/test/selenium.py @@ -1,8 +1,11 @@ import sys import unittest from contextlib import contextmanager +from functools import wraps +from pathlib import Path -from django.test import LiveServerTestCase, tag +from django.conf import settings +from django.test import LiveServerTestCase, override_settings, tag from django.utils.functional import classproperty from django.utils.module_loading import import_string from django.utils.text import capfirst @@ -116,6 +119,30 @@ class ChangeWindowSize: class SeleniumTestCase(LiveServerTestCase, metaclass=SeleniumTestCaseBase): implicit_wait = 10 external_host = None + screenshots = False + + @classmethod + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + if not cls.screenshots: + return + + for name, func in list(cls.__dict__.items()): + if not hasattr(func, "_screenshot_cases"): + continue + # Remove the main test. + delattr(cls, name) + # Add separate tests for each screenshot type. + for screenshot_case in getattr(func, "_screenshot_cases"): + + @wraps(func) + def test(self, *args, _func=func, _case=screenshot_case, **kwargs): + with getattr(self, _case)(): + return _func(self, *args, **kwargs) + + test.__name__ = f"{name}_{screenshot_case}" + test.__qualname__ = f"{test.__qualname__}_{screenshot_case}" + setattr(cls, test.__name__, test) @classproperty def live_server_url(cls): @@ -147,6 +174,30 @@ class SeleniumTestCase(LiveServerTestCase, metaclass=SeleniumTestCaseBase): with ChangeWindowSize(360, 800, self.selenium): yield + @contextmanager + def rtl(self): + with self.desktop_size(): + with override_settings(LANGUAGE_CODE=settings.LANGUAGES_BIDI[-1]): + yield + + @contextmanager + def dark(self): + # Navigate to a page before executing a script. + self.selenium.get(self.live_server_url) + self.selenium.execute_script("localStorage.setItem('theme', 'dark');") + with self.desktop_size(): + try: + yield + finally: + self.selenium.execute_script("localStorage.removeItem('theme');") + + def take_screenshot(self, name): + if not self.screenshots: + return + path = Path.cwd() / "screenshots" / f"{self._testMethodName}-{name}.png" + path.parent.mkdir(exist_ok=True, parents=True) + self.selenium.save_screenshot(path) + @classmethod def _quit_selenium(cls): # quit() the WebDriver before attempting to terminate and join the @@ -163,3 +214,15 @@ class SeleniumTestCase(LiveServerTestCase, metaclass=SeleniumTestCaseBase): yield finally: self.selenium.implicitly_wait(self.implicit_wait) + + +def screenshot_cases(method_names): + if isinstance(method_names, str): + method_names = method_names.split(",") + + def wrapper(func): + func._screenshot_cases = method_names + setattr(func, "tags", {"screenshot"}.union(getattr(func, "tags", set()))) + return func + + return wrapper diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index 4f5cbeb125..1e86a5802d 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -271,6 +271,37 @@ faster and more stable. Add the ``--headless`` option to enable this mode. .. _selenium.webdriver: https://github.com/SeleniumHQ/selenium/tree/trunk/py/selenium/webdriver +For testing changes to the admin UI, the selenium tests can be run with the +``--screenshots`` option enabled. Screenshots will be saved to the +``tests/screenshots/`` directory. + +To define when screenshots should be taken during a selenium test, the test +class must use the ``@django.test.selenium.screenshot_cases`` decorator with a +list of supported screenshot types (``"desktop_size"``, ``"mobile_size"``, +``"small_screen_size"``, ``"rtl"``, and ``"dark"``). It can then call +``self.take_screenshot("unique-screenshot-name")`` at the desired point to +generate the screenshots. For example:: + + from django.test.selenium import SeleniumTestCase, screenshot_cases + from django.urls import reverse + + + class SeleniumTests(SeleniumTestCase): + @screenshot_cases(["desktop_size", "mobile_size", "rtl", "dark"]) + def test_login_button_centered(self): + self.selenium.get(self.live_server_url + reverse("admin:login")) + self.take_screenshot("login") + ... + +This generates multiple screenshots of the login page - one for a desktop +screen, one for a mobile screen, one for right-to-left languages on desktop, +and one for the dark mode on desktop. + +.. versionchanged:: 5.1 + + The ``--screenshots`` option and ``@screenshot_cases`` decorator were + added. + .. _running-unit-tests-dependencies: Running all the tests diff --git a/docs/releases/5.1.txt b/docs/releases/5.1.txt index aa8fe80aac..bdaeda1b8d 100644 --- a/docs/releases/5.1.txt +++ b/docs/releases/5.1.txt @@ -206,6 +206,9 @@ Tests :meth:`~django.test.SimpleTestCase.assertInHTML` assertions now add haystacks to assertion error messages. +* Django test runner now supports ``--screenshots`` option to save screenshots + for Selenium tests. + URLs ~~~~ diff --git a/tests/admin_views/tests.py b/tests/admin_views/tests.py index 4fddace5ea..19877a933e 100644 --- a/tests/admin_views/tests.py +++ b/tests/admin_views/tests.py @@ -35,6 +35,7 @@ from django.test import ( override_settings, skipUnlessDBFeature, ) +from django.test.selenium import screenshot_cases from django.test.utils import override_script_prefix from django.urls import NoReverseMatch, resolve, reverse from django.utils import formats, translation @@ -5732,6 +5733,7 @@ class SeleniumTests(AdminSeleniumTestCase): title="A Long Title", published=True, slug="a-long-title" ) + @screenshot_cases(["desktop_size", "mobile_size", "rtl", "dark"]) def test_login_button_centered(self): from selenium.webdriver.common.by import By @@ -5743,6 +5745,7 @@ class SeleniumTests(AdminSeleniumTestCase): ) - (offset_left + button.get_property("offsetWidth")) # Use assertAlmostEqual to avoid pixel rounding errors. self.assertAlmostEqual(offset_left, offset_right, delta=3) + self.take_screenshot("login") def test_prepopulated_fields(self): """ @@ -6017,6 +6020,7 @@ class SeleniumTests(AdminSeleniumTestCase): self.assertEqual(slug1, "this-is-the-main-name-the-best-2012-02-18") self.assertEqual(slug2, "option-two-this-is-the-main-name-the-best") + @screenshot_cases(["desktop_size", "mobile_size", "dark"]) def test_collapsible_fieldset(self): """ The 'collapse' class in fieldsets definition allows to @@ -6031,12 +6035,15 @@ class SeleniumTests(AdminSeleniumTestCase): self.live_server_url + reverse("admin:admin_views_article_add") ) self.assertFalse(self.selenium.find_element(By.ID, "id_title").is_displayed()) + self.take_screenshot("collapsed") self.selenium.find_elements(By.LINK_TEXT, "Show")[0].click() self.assertTrue(self.selenium.find_element(By.ID, "id_title").is_displayed()) self.assertEqual( self.selenium.find_element(By.ID, "fieldsetcollapser0").text, "Hide" ) + self.take_screenshot("expanded") + @screenshot_cases(["desktop_size", "mobile_size", "rtl", "dark"]) def test_selectbox_height_collapsible_fieldset(self): from selenium.webdriver.common.by import By @@ -6047,7 +6054,7 @@ class SeleniumTests(AdminSeleniumTestCase): ) url = self.live_server_url + reverse("admin7:admin_views_pizza_add") self.selenium.get(url) - self.selenium.find_elements(By.LINK_TEXT, "Show")[0].click() + self.selenium.find_elements(By.ID, "fieldsetcollapser0")[0].click() from_filter_box = self.selenium.find_element(By.ID, "id_toppings_filter") from_box = self.selenium.find_element(By.ID, "id_toppings_from") to_filter_box = self.selenium.find_element(By.ID, "id_toppings_filter_selected") @@ -6062,7 +6069,9 @@ class SeleniumTests(AdminSeleniumTestCase): + from_box.get_property("offsetHeight") ), ) + self.take_screenshot("selectbox-collapsible") + @screenshot_cases(["desktop_size", "mobile_size", "rtl", "dark"]) def test_selectbox_height_not_collapsible_fieldset(self): from selenium.webdriver.common.by import By @@ -6091,7 +6100,9 @@ class SeleniumTests(AdminSeleniumTestCase): + from_box.get_property("offsetHeight") ), ) + self.take_screenshot("selectbox-non-collapsible") + @screenshot_cases(["desktop_size", "mobile_size", "rtl", "dark"]) def test_first_field_focus(self): """JavaScript-assisted auto-focus on first usable form field.""" from selenium.webdriver.common.by import By @@ -6108,6 +6119,7 @@ class SeleniumTests(AdminSeleniumTestCase): self.selenium.switch_to.active_element, self.selenium.find_element(By.ID, "id_name"), ) + self.take_screenshot("focus-single-widget") # First form field has a MultiWidget with self.wait_page_loaded(): @@ -6118,6 +6130,7 @@ class SeleniumTests(AdminSeleniumTestCase): self.selenium.switch_to.active_element, self.selenium.find_element(By.ID, "id_start_date_0"), ) + self.take_screenshot("focus-multi-widget") def test_cancel_delete_confirmation(self): "Cancelling the deletion of an object takes the user back one page." diff --git a/tests/runtests.py b/tests/runtests.py index b5f499fe7e..e03966410b 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -26,7 +26,7 @@ else: from django.db import connection, connections from django.test import TestCase, TransactionTestCase from django.test.runner import get_max_test_processes, parallel_type - from django.test.selenium import SeleniumTestCaseBase + from django.test.selenium import SeleniumTestCase, SeleniumTestCaseBase from django.test.utils import NullTimeKeeper, TimeKeeper, get_runner from django.utils.deprecation import RemovedInDjango60Warning from django.utils.log import DEFAULT_LOGGING @@ -598,6 +598,11 @@ if __name__ == "__main__": metavar="BROWSERS", help="A comma-separated list of browsers to run the Selenium tests against.", ) + parser.add_argument( + "--screenshots", + action="store_true", + help="Take screenshots during selenium tests to capture the user interface.", + ) parser.add_argument( "--headless", action="store_true", @@ -699,6 +704,10 @@ if __name__ == "__main__": ) if using_selenium_hub and not options.external_host: parser.error("--selenium-hub and --external-host must be used together.") + if options.screenshots and not options.selenium: + parser.error("--screenshots require --selenium to be used.") + if options.screenshots and options.tags: + parser.error("--screenshots and --tag are mutually exclusive.") # Allow including a trailing slash on app_labels for tab completion convenience options.modules = [os.path.normpath(labels) for labels in options.modules] @@ -748,6 +757,9 @@ if __name__ == "__main__": SeleniumTestCaseBase.external_host = options.external_host SeleniumTestCaseBase.headless = options.headless SeleniumTestCaseBase.browsers = options.selenium + if options.screenshots: + options.tags = ["screenshot"] + SeleniumTestCase.screenshots = options.screenshots if options.bisect: bisect_tests(