Fixed #29942 -- Restored source file linking in docs by using the Sphinx linkcode ext.

Co-authored-by: David Smith <smithdc@gmail.com>
Co-authored-by: Natalia <124304+nessita@users.noreply.github.com>

django/forms/formsets.py

@@ -1,6 +1,6 @@
 from django.core.exceptions import ValidationError
-from django.forms import Form
 from django.forms.fields import BooleanField, IntegerField
+from django.forms.forms import Form
 from django.forms.renderers import get_default_renderer
 from django.forms.utils import ErrorList, RenderableFormMixin
 from django.forms.widgets import CheckboxInput, HiddenInput, NumberInput

docs/_ext/github_links.py

@@ -0,0 +1,149 @@
+import ast
+import functools
+import importlib.util
+import pathlib
+
+
+class CodeLocator(ast.NodeVisitor):
+    def __init__(self):
+        super().__init__()
+        self.current_path = []
+        self.node_line_numbers = {}
+        self.import_locations = {}
+
+    @classmethod
+    def from_code(cls, code):
+        tree = ast.parse(code)
+        locator = cls()
+        locator.visit(tree)
+        return locator
+
+    def visit_node(self, node):
+        self.current_path.append(node.name)
+        self.node_line_numbers[".".join(self.current_path)] = node.lineno
+        self.generic_visit(node)
+        self.current_path.pop()
+
+    def visit_FunctionDef(self, node):
+        self.visit_node(node)
+
+    def visit_ClassDef(self, node):
+        self.visit_node(node)
+
+    def visit_ImportFrom(self, node):
+        for alias in node.names:
+            if alias.asname:
+                # Exclude linking aliases (`import x as y`) to avoid confusion
+                # when clicking a source link to a differently named entity.
+                continue
+            if alias.name == "*":
+                # Resolve wildcard imports.
+                file = module_name_to_file_path(node.module)
+                file_contents = file.read_text(encoding="utf-8")
+                locator = CodeLocator.from_code(file_contents)
+                self.import_locations |= locator.import_locations
+                self.import_locations |= {
+                    n: node.module for n in locator.node_line_numbers if "." not in n
+                }
+            else:
+                self.import_locations[alias.name] = ("." * node.level) + (
+                    node.module or ""
+                )
+
+
+@functools.lru_cache(maxsize=1024)
+def get_locator(file):
+    file_contents = file.read_text(encoding="utf-8")
+    return CodeLocator.from_code(file_contents)
+
+
+class CodeNotFound(Exception):
+    pass
+
+
+def module_name_to_file_path(module_name):
+    # Avoid importlib machinery as locating a module involves importing its
+    # parent, which would trigger import side effects.
+
+    for suffix in [".py", "/__init__.py"]:
+        file_path = pathlib.Path(__file__).parents[2] / (
+            module_name.replace(".", "/") + suffix
+        )
+        if file_path.exists():
+            return file_path
+
+    raise CodeNotFound
+
+
+def get_path_and_line(module, fullname):
+    path = module_name_to_file_path(module_name=module)
+
+    locator = get_locator(path)
+
+    lineno = locator.node_line_numbers.get(fullname)
+
+    if lineno is not None:
+        return path, lineno
+
+    imported_object = fullname.split(".", maxsplit=1)[0]
+    try:
+        imported_path = locator.import_locations[imported_object]
+    except KeyError:
+        raise CodeNotFound
+
+    # From a statement such as:
+    # from . import y.z
+    # - either y.z might be an object in the parent module
+    # - or y might be a module, and z be an object in y
+    # also:
+    # - either the current file is x/__init__.py, and z would be in x.y
+    # - or the current file is x/a.py, and z would be in x.a.y
+    if path.name != "__init__.py":
+        # Look in parent module
+        module = module.rsplit(".", maxsplit=1)[0]
+    try:
+        imported_module = importlib.util.resolve_name(
+            name=imported_path, package=module
+        )
+    except ImportError as error:
+        raise ImportError(
+            f"Could not import '{imported_path}' in '{module}'."
+        ) from error
+    try:
+        return get_path_and_line(module=imported_module, fullname=fullname)
+    except CodeNotFound:
+        if "." not in fullname:
+            raise
+
+        first_element, remainder = fullname.rsplit(".", maxsplit=1)
+        # Retrying, assuming the first element of the fullname is a module.
+        return get_path_and_line(
+            module=f"{imported_module}.{first_element}", fullname=remainder
+        )
+
+
+def get_branch(version, next_version):
+    if version == next_version:
+        return "main"
+    else:
+        return f"stable/{version}.x"
+
+
+def github_linkcode_resolve(domain, info, *, version, next_version):
+    if domain != "py":
+        return None
+
+    if not (module := info["module"]):
+        return None
+
+    try:
+        path, lineno = get_path_and_line(module=module, fullname=info["fullname"])
+    except CodeNotFound:
+        return None
+
+    branch = get_branch(version=version, next_version=next_version)
+    relative_path = path.relative_to(pathlib.Path(__file__).parents[2])
+    # Use "/" explicitely to join the path parts since str(file), on Windows,
+    # uses the Windows path separator which is incorrect for URLs.
+    url_path = "/".join(relative_path.parts)
+    return f"https://github.com/django/django/blob/{branch}/{url_path}#L{lineno}"

docs/conf.py

@@ -9,6 +9,7 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+import functools
 import sys
 from os.path import abspath, dirname, join
 
@@ -29,6 +30,10 @@ sys.path.insert(1, dirname(dirname(abspath(__file__))))
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.append(abspath(join(dirname(__file__), "_ext")))
 
+# Use the module to GitHub url resolver, but import it after the _ext directoy
+# it lives in has been added to sys.path.
+import github_links  # NOQA
+
 # -- General configuration -----------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -40,8 +45,8 @@ extensions = [
     "djangodocs",
     "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
-    "sphinx.ext.viewcode",
     "sphinx.ext.autosectionlabel",
+    "sphinx.ext.linkcode",
 ]
 
 # AutosectionLabel settings.
@@ -432,3 +437,9 @@ epub_cover = ("", "epub-cover.html")
 
 # If false, no index is generated.
 # epub_use_index = True
+
+linkcode_resolve = functools.partial(
+    github_links.github_linkcode_resolve,
+    version=version,
+    next_version=django_next_version,
+)

tests/sphinx/test_github_links.py

@@ -0,0 +1,211 @@
+import pathlib
+import sys
+
+from django.test import SimpleTestCase
+
+
+def last_n_parts(path, n):
+    return "/".join(path.parts[-n:])
+
+
+# The import must happen at the end of setUpClass, so it can't be imported at
+# the top of the file.
+github_links = None
+
+
+class GitHubLinkTests(SimpleTestCase):
+    @classmethod
+    def setUpClass(cls):
+        # The file implementing the code under test is in the docs folder and
+        # is not part of the Django package. This means it cannot be imported
+        # through standard means. Include its parent in the pythonpath for the
+        # duration of the tests to allow the code to be imported.
+        cls.ext_path = str((pathlib.Path(__file__).parents[2] / "docs/_ext").resolve())
+        sys.path.insert(0, cls.ext_path)
+        cls.addClassCleanup(sys.path.remove, cls.ext_path)
+        cls.addClassCleanup(sys.modules.pop, "github_links", None)
+        # Linters/IDEs may not be able to detect this as a valid import.
+        import github_links as _github_links
+
+        global github_links
+        github_links = _github_links
+
+    def test_code_locator(self):
+        locator = github_links.CodeLocator.from_code(
+            """
+from a import b, c
+from .d import e, f as g
+
+def h():
+    pass
+
+class I:
+    def j(self):
+        pass"""
+        )
+
+        self.assertEqual(locator.node_line_numbers, {"h": 5, "I": 8, "I.j": 9})
+        self.assertEqual(locator.import_locations, {"b": "a", "c": "a", "e": ".d"})
+
+    def test_module_name_to_file_path_package(self):
+        path = github_links.module_name_to_file_path("django")
+
+        self.assertEqual(last_n_parts(path, 2), "django/__init__.py")
+
+    def test_module_name_to_file_path_module(self):
+        path = github_links.module_name_to_file_path("django.shortcuts")
+
+        self.assertEqual(last_n_parts(path, 2), "django/shortcuts.py")
+
+    def test_get_path_and_line_class(self):
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module", fullname="MyClass"
+        )
+
+        self.assertEqual(
+            last_n_parts(path, 5), "tests/sphinx/testdata/package/module.py"
+        )
+        self.assertEqual(line, 12)
+
+    def test_get_path_and_line_func(self):
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module", fullname="my_function"
+        )
+
+        self.assertEqual(
+            last_n_parts(path, 5), "tests/sphinx/testdata/package/module.py"
+        )
+        self.assertEqual(line, 24)
+
+    def test_get_path_and_line_method(self):
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module", fullname="MyClass.my_method"
+        )
+
+        self.assertEqual(
+            last_n_parts(path, 5), "tests/sphinx/testdata/package/module.py"
+        )
+        self.assertEqual(line, 16)
+
+    def test_get_path_and_line_cached_property(self):
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module",
+            fullname="MyClass.my_cached_property",
+        )
+
+        self.assertEqual(
+            last_n_parts(path, 5), "tests/sphinx/testdata/package/module.py"
+        )
+        self.assertEqual(line, 20)
+
+    def test_get_path_and_line_forwarded_import(self):
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module", fullname="MyOtherClass"
+        )
+
+        self.assertEqual(
+            last_n_parts(path, 5), "tests/sphinx/testdata/package/other_module.py"
+        )
+        self.assertEqual(line, 1)
+
+    def test_get_path_and_line_wildcard_import(self):
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module", fullname="WildcardClass"
+        )
+
+        self.assertEqual(
+            last_n_parts(path, 5),
+            "tests/sphinx/testdata/package/wildcard_module.py",
+        )
+        self.assertEqual(line, 4)
+
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module",
+            fullname="WildcardMixin",
+        )
+        self.assertEqual(
+            last_n_parts(path, 5),
+            "tests/sphinx/testdata/package/wildcard_base.py",
+        )
+        self.assertEqual(line, 1)
+
+    def test_get_path_and_line_forwarded_import_module(self):
+        path, line = github_links.get_path_and_line(
+            module="tests.sphinx.testdata.package.module",
+            fullname="other_module.MyOtherClass",
+        )
+
+        self.assertEqual(
+            last_n_parts(path, 5), "tests/sphinx/testdata/package/other_module.py"
+        )
+        self.assertEqual(line, 1)
+
+    def test_get_branch_stable(self):
+        branch = github_links.get_branch(version="2.2", next_version="3.2")
+        self.assertEqual(branch, "stable/2.2.x")
+
+    def test_get_branch_latest(self):
+        branch = github_links.get_branch(version="3.2", next_version="3.2")
+        self.assertEqual(branch, "main")
+
+    def test_github_linkcode_resolve_unspecified_domain(self):
+        domain = "unspecified"
+        info = {}
+        self.assertIsNone(
+            github_links.github_linkcode_resolve(
+                domain, info, version="3.2", next_version="3.2"
+            )
+        )
+
+    def test_github_linkcode_resolve_unspecified_info(self):
+        domain = "py"
+        info = {"module": None, "fullname": None}
+        self.assertIsNone(
+            github_links.github_linkcode_resolve(
+                domain, info, version="3.2", next_version="3.2"
+            )
+        )
+
+    def test_github_linkcode_resolve_not_found(self):
+        info = {
+            "module": "foo.bar.baz.hopefully_non_existant_module",
+            "fullname": "MyClass",
+        }
+        self.assertIsNone(
+            github_links.github_linkcode_resolve(
+                "py", info, version="3.2", next_version="3.2"
+            )
+        )
+
+    def test_github_linkcode_resolve_link_to_object(self):
+        info = {
+            "module": "tests.sphinx.testdata.package.module",
+            "fullname": "MyClass",
+        }
+        self.assertEqual(
+            github_links.github_linkcode_resolve(
+                "py", info, version="3.2", next_version="3.2"
+            ),
+            "https://github.com/django/django/blob/main/tests/sphinx/"
+            "testdata/package/module.py#L12",
+        )
+
+    def test_github_linkcode_resolve_link_to_class_older_version(self):
+        info = {
+            "module": "tests.sphinx.testdata.package.module",
+            "fullname": "MyClass",
+        }
+        self.assertEqual(
+            github_links.github_linkcode_resolve(
+                "py", info, version="2.2", next_version="3.2"
+            ),
+            "https://github.com/django/django/blob/stable/2.2.x/tests/sphinx/"
+            "testdata/package/module.py#L12",
+        )
+
+    def test_import_error(self):
+        msg = "Could not import '.....test' in 'tests.sphinx.testdata.package'."
+        with self.assertRaisesMessage(ImportError, msg):
+            github_links.get_path_and_line(
+                module="tests.sphinx.testdata.package.import_error", fullname="Test"
+            )

tests/sphinx/testdata/package/__init__.py

@@ -0,0 +1,2 @@
+# This file should never get imported. If it is, then something failed already.
+raise Exception

tests/sphinx/testdata/package/import_error.py

@@ -0,0 +1,5 @@
+from .....test import Test  # noqa
+
+
+class MyClass:
+    pass

tests/sphinx/testdata/package/module.py

@@ -0,0 +1,25 @@
+"""
+Example docstring
+"""
+
+from django.utils.functional import cached_property
+from tests.sphinx.testdata.package.wildcard_module import *  # noqa
+
+from . import other_module  # noqa
+from .other_module import MyOtherClass  # noqa
+
+
+class MyClass(object):
+    def __init__(self):
+        pass
+
+    def my_method(self):
+        pass
+
+    @cached_property
+    def my_cached_property(self):
+        pass
+
+
+def my_function(self):
+    pass

tests/sphinx/testdata/package/other_module.py

@@ -0,0 +1,2 @@
+class MyOtherClass:
+    pass

tests/sphinx/testdata/package/wildcard_base.py

@@ -0,0 +1,2 @@
+class WildcardMixin:
+    pass

tests/sphinx/testdata/package/wildcard_module.py

@@ -0,0 +1,5 @@
+from .wildcard_base import WildcardMixin  # noqa
+
+
+class WildcardClass:
+    pass