From b8cb5471301a782646d80ba06afeee5dbb9299bf Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 5 May 2022 13:03:48 -0700 Subject: [PATCH 01/26] Add object-ids and nonodeid options to Python domain The `nonodeid` directive option replaces the `python_qualify_parameter_ids` config option. This options are primarily intended for use by the Python autosummary extension to be added in a subsequent commit. --- docs/api.rst | 2 + docs/python.rst | 90 +++++++++++++++++-- sphinx_immaterial/python_domain_fixes.py | 109 +++++++++++++++++++++-- 3 files changed, 187 insertions(+), 14 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 687a2cc37..3c8538203 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -141,6 +141,8 @@ Other options described elsewhere include: - :objconf:`wrap_signatures_column_limit` - :objconf:`clang_format_style` +.. _object-toc-icons: + Table of contents icons ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/python.rst b/docs/python.rst index f33439ae9..c8b479377 100644 --- a/docs/python.rst +++ b/docs/python.rst @@ -79,12 +79,88 @@ Python domain customization The concise syntax is non-standard and not accepted by Python type checkers. -.. confval:: python_qualify_parameter_ids +Overloaded functions +-------------------- - Specifies whether function parameters should be assigned fully-qualified ids - (for cross-linking purposes) of the form ``.`` based - on the id of the parent declaration. +The Sphinx Python domain supports documenting multiple signatures together as +part of the same object description: - If set to :python:`False`, instead the shorter unqualified id - ``p-`` is used. This option should only be set to - :python:`False` if each Python declaration is on a separate page. +.. rst-example:: + + + .. py:function:: overload_example1(a: int) -> int + overload_example1(a: float) -> float + overload_example1(a: str) -> str + + Does something with an `int`, `float`, or `str`. + +However, it does not provide a way to document each overload with a separate +description, except by using the ``:noindex:`` option to avoid a warning from +duplicate definitions. + +This theme extends the Python domain directives with an ``:object-ids:`` option to +allow multiple overloads of a given function to be documented separately: + +The value of the ``:object-ids:`` option must be a JSON-encoded array of +strings, where each string specifies the full object name (including module +name) to use for each signature. The object ids must start with the actual +module name, if any, but the remainder of the id need not match the name +specified in the signature. + +.. rst-example:: + + .. py:function:: overload_example2(a: int) -> int + overload_example2(a: float) -> float + :object-ids: ["overload_example2(int)", "overload_example2(float)"] + + Does something with an `int` or `float`. + + .. py:function:: overload_example2(a: str) -> str + + :object-ids: ["overload_example2(str)"] + + Does something with a `str`. + +If this option is specified, and :objconf:`generate_synopses` is enabled, then a +synopsis will be stored even if ``:noindex`` is also specified. + +Separate page for object description +------------------------------------ + +Normally, the Python domain generates an ``id`` attribute for each object +description based on its full name. This may be used in a URL to target a +specific object description, e.g. ``api/tensorstore.html#tensorstore.IndexDomain``. + +If an entire page is dedicated to a single object description, this ``id`` is +essentially redundant, +e.g. ``api/tensorstore.IndexDomain.html#tensorstore.IndexDomain``. + +This theme extends the Python domain directives (as well as the corresponding +``auto`` directives provided by the `sphinx.ext.autodoc` extension) +with a ``:nonodeid:`` option: + +.. code-block:: python + + .. py:function:: func(a: int) -> int + :nonodeid: + +If this option is specified, the object description itself will not have an +``id``, and any cross references to the object will simply target the page. +Additionally, any table of contents entry for the page will have an associated +:ref:`icon` if one has been configured for the object type. + +.. note:: + + Sphinx itself supports two related options for Python domain directives: + + - :rst:`:noindex:`: prevents the creation of a cross-reference target + entirely. The object will not appear in search results (except through + text matches). + + - :rst:`:noindexentry:`: prevents inclusion of the object in the "general + index" (not normally useful with this theme anyway). A cross-reference + target is still created, and the object still appears in search results. + + In contrast, if the :rst:`:nonodeid:` option is specified, a cross-reference + target is still created, and the object is still included in search results. + However, any cross references to the object will link to the containing page. diff --git a/sphinx_immaterial/python_domain_fixes.py b/sphinx_immaterial/python_domain_fixes.py index 6d082fafd..67c91de3d 100644 --- a/sphinx_immaterial/python_domain_fixes.py +++ b/sphinx_immaterial/python_domain_fixes.py @@ -1,5 +1,6 @@ """Fixes for the Python domain.""" +import json from typing import ( cast, Sequence, @@ -441,7 +442,7 @@ def _add_parameter_documentation_ids( noindex: bool, ) -> None: - qualify_parameter_ids = env.config.python_qualify_parameter_ids + qualify_parameter_ids = "nonodeid" not in directive.options param_options = apidoc_formatting.get_object_description_options( env, "py", "parameter" @@ -633,12 +634,46 @@ def after_content(self: PyObject) -> None: ).parent signodes = obj_desc.children[:-1] + py = cast(PythonDomain, self.env.get_domain("py")) + + def strip_object_entry_node_id(existing_node_id: str, object_id: str): + obj = py.objects.get(object_id) + if ( + obj is None + or obj.node_id != existing_node_id + or obj.docname != self.env.docname + ): + return + py.objects[object_id] = obj._replace(node_id="") + + nonodeid = "nonodeid" in self.options + canonical_name = self.options.get("canonical") + noindexentry = "noindexentry" in self.options + noindex = "noindex" in self.options + symbols = [] for signode in cast(List[docutils.nodes.Element], signodes): modname = signode["module"] fullname = signode["fullname"] - symbols.append((modname + "." if modname else "") + fullname) - noindex = "noindex" in self.options + symbol = (modname + "." if modname else "") + fullname + symbols.append(symbol) + if nonodeid and signode["ids"]: + orig_node_id = signode["ids"][0] + signode["ids"] = [] + strip_object_entry_node_id(orig_node_id, symbol) + if canonical_name: + strip_object_entry_node_id(orig_node_id, canonical_name) + + if noindexentry: + entries = self.indexnode["entries"] + new_entries = [] + for entry in entries: + new_entry = list(entry) + if new_entry[2] == orig_node_id: + new_entry[2] = "" + new_entries.append(tuple(new_entry)) + self.indexnode["entries"] = new_entries + if not symbols: return if self.objtype in ("class", "exception"): @@ -669,7 +704,6 @@ def after_content(self: PyObject) -> None: ) if not synopsis: return - py = cast(PythonDomain, self.env.get_domain("py")) for symbol in symbols: py.data["synopses"][symbol] = synopsis @@ -701,6 +735,69 @@ def get_object_synopses( ) +def _monkey_patch_python_domain_to_support_object_ids(): + for object_class in sphinx.domains.python.PythonDomain.directives.values(): + object_class.option_spec["object-ids"] = json.loads + object_class.option_spec["nonodeid"] = docutils.parsers.rst.directives.flag + + passthrough_options = ("object-ids", "nonodeid") + + orig_add_directive_header = sphinx.ext.autodoc.Documenter.add_directive_header + + def add_directive_header(self: sphinx.ext.autodoc.Documenter, sig: str) -> None: + orig_add_directive_header(self, sig) + for option_name in passthrough_options: + if option_name not in self.options: + continue + value = self.options[option_name] + self.add_line(f" :{option_name}: {value}", self.get_sourcename()) + + sphinx.ext.autodoc.Documenter.add_directive_header = add_directive_header + + orig_handle_signature = sphinx.domains.python.PyObject.handle_signature + + def handle_signature( + self: sphinx.domains.python.PyObject, + sig: str, + signode: sphinx.addnodes.desc_signature, + ) -> Tuple[str, str]: + fullname, prefix = orig_handle_signature(self, sig, signode) + object_ids = self.options.get("object-ids") + if object_ids is not None: + signature_index = getattr(self, "_signature_index", 0) + setattr(self, "_signature_index", signature_index + 1) + modname = signode["module"] + if modname: + modname += "." + else: + modname = "" + if signature_index >= len(object_ids): + logger.warning( + "Not enough object-ids %r specified for %r", + object_ids, + modname + signode["fullname"], + location=self.get_source_info(), + ) + else: + object_id = object_ids[signature_index] + if object_id.startswith(modname): + fullname = object_id[len(modname) :] + signode["fullname"] = fullname + else: + logger.warning( + "object-id %r for %r does not start with module name %r", + object_id, + signode["fullname"], + modname, + location=self.get_source_info(), + ) + return fullname, prefix + + sphinx.domains.python.PyObject.handle_signature = handle_signature + + + + def setup(app: sphinx.application.Sphinx): _monkey_patch_python_doc_fields() _monkey_patch_python_parse_annotation() @@ -716,13 +813,11 @@ def setup(app: sphinx.application.Sphinx): _monkey_patch_python_domain_to_deprioritize_params_in_search() _monkey_patch_python_domain_to_add_object_synopses_to_references() _monkey_patch_python_domain_to_support_synopses() + _monkey_patch_python_domain_to_support_object_ids() sphinx.domains.python.PythonDomain.initial_data["synopses"] = {} # name -> synopsis app.add_role_to_domain("py", "param", PyParamXRefRole()) - app.add_config_value( - "python_qualify_parameter_ids", default=True, rebuild="env", types=(bool,) - ) return { "parallel_read_safe": True, From 431a6be3658722dc21ad479b7bedc327ba4db130 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 5 May 2022 13:09:16 -0700 Subject: [PATCH 02/26] Fix autodoc_property_type module to avoid duplicate :type: option --- sphinx_immaterial/autodoc_property_type.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sphinx_immaterial/autodoc_property_type.py b/sphinx_immaterial/autodoc_property_type.py index c1ab127fa..fe8426bb8 100644 --- a/sphinx_immaterial/autodoc_property_type.py +++ b/sphinx_immaterial/autodoc_property_type.py @@ -49,12 +49,22 @@ def import_object(self: PropertyDocumenter, raiseerror: bool = False) -> bool: old_add_directive_header = PropertyDocumenter.add_directive_header def add_directive_header(self, sig: str) -> None: + start_line = len(self.directive.result.data) old_add_directive_header(self, sig) # Check for return annotation retann = self.retann or _get_property_return_type(self.object) - if retann is not None: - self.add_line(" :type: " + retann, self.get_sourcename()) + if retann is None: + return + + # Check if type annotation has already been added. + type_line_prefix = self.indent + " :type: " + for line in self.directive.result.data[start_line:]: + if line.startswith(type_line_prefix): + return + + # Type annotation not already added. + self.add_line(" :type: " + retann, self.get_sourcename()) PropertyDocumenter.add_directive_header = add_directive_header From 4fd5e3688bb4fcdefb5de2b953eca3bed0880400 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 5 May 2022 13:22:51 -0700 Subject: [PATCH 03/26] Add python_strip_{self,return}_type_annotations config options --- docs/python.rst | 49 +++++++++++++++++++ sphinx_immaterial/python_domain_fixes.py | 60 ++++++++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/docs/python.rst b/docs/python.rst index c8b479377..a4b2398df 100644 --- a/docs/python.rst +++ b/docs/python.rst @@ -79,6 +79,55 @@ Python domain customization The concise syntax is non-standard and not accepted by Python type checkers. +.. confval:: python_strip_self_type_annotations + + Strip type annotations from the initial :python:`self` parameter of methods. + + Since the :python:`self` type is usually evident from the context, removing + them may improve readability of the documentation. + + .. note:: + + This option is useful when generating documentation from `pybind11 + `__ + modules, as pybind11 adds these type annotations. + + .. rst-example:: + + .. py:class:: Example + :noindex: + + .. py:method:: foo(self: Example, a: int) -> int + :noindex: + + Does something with the object. + +.. confval:: python_strip_return_type_annotations + + Regular expression pattern that matches the full name (including module) of + functions for which any return type annotations should be stripped. + + Setting this to `None` disables stripping of return type annotations. + + By default, the return type is stripped from :python:`__init__` and + :python:`__setitem__` functions (which usually return :python:`None`). + + .. note:: + + This option is useful when generating documentation from `pybind11 + `__ + modules, as pybind11 adds these type annotations. + + .. rst-example:: + + .. py:class:: Example + :noindex: + + .. py:method:: __setitem__(self, a: int, b: int) -> None + :noindex: + + Does something with the object. + Overloaded functions -------------------- diff --git a/sphinx_immaterial/python_domain_fixes.py b/sphinx_immaterial/python_domain_fixes.py index 67c91de3d..bd4302fb2 100644 --- a/sphinx_immaterial/python_domain_fixes.py +++ b/sphinx_immaterial/python_domain_fixes.py @@ -1,6 +1,7 @@ """Fixes for the Python domain.""" import json +import re from typing import ( cast, Sequence, @@ -735,6 +736,40 @@ def get_object_synopses( ) +def _maybe_strip_type_annotations( + app: sphinx.application.Sphinx, + domain: str, + objtype: str, + contentnode: sphinx.addnodes.desc_content, +) -> None: + if domain != "py": + return + obj_desc = contentnode.parent + assert isinstance(obj_desc, sphinx.addnodes.desc) + strip_self_type_annotations = app.config.python_strip_self_type_annotations + strip_return_type_annotations = app.config.python_strip_return_type_annotations + for signode in obj_desc[:-1]: + assert isinstance(signode, sphinx.addnodes.desc_signature) + if strip_self_type_annotations: + for param in signode.traverse(condition=sphinx.addnodes.desc_parameter): + if param.children[0].astext() == "self": + # Remove any annotations on `self` + del param.children[1:] + break + if strip_return_type_annotations is not None: + fullname = signode.get("fullname") + if fullname is None: + # Python domain failed to parse the signature. Just ignore it. + continue + modname = signode["module"] + if modname: + fullname = modname + "." + fullname + if strip_return_type_annotations.fullmatch(fullname): + # Remove return type. + for node in signode.traverse(condition=sphinx.addnodes.desc_returns): + node.parent.remove(node) + + def _monkey_patch_python_domain_to_support_object_ids(): for object_class in sphinx.domains.python.PythonDomain.directives.values(): object_class.option_spec["object-ids"] = json.loads @@ -796,6 +831,21 @@ def handle_signature( sphinx.domains.python.PyObject.handle_signature = handle_signature +def _config_inited( + app: sphinx.application.Sphinx, config: sphinx.config.Config +) -> None: + + if ( + config.python_strip_self_type_annotations + or config.python_strip_return_type_annotations + ): + if isinstance(config.python_strip_return_type_annotations, str): + setattr( + config, + "python_strip_return_type_annotations", + re.compile(config.python_strip_return_type_annotations), + ) + app.connect("object-description-transform", _maybe_strip_type_annotations) def setup(app: sphinx.application.Sphinx): @@ -818,6 +868,16 @@ def setup(app: sphinx.application.Sphinx): sphinx.domains.python.PythonDomain.initial_data["synopses"] = {} # name -> synopsis app.add_role_to_domain("py", "param", PyParamXRefRole()) + app.add_config_value( + "python_strip_self_type_annotations", default=True, rebuild="env", types=(bool,) + ) + app.add_config_value( + "python_strip_return_type_annotations", + default=r".*.(__setitem__|__init__)", + rebuild="env", + types=(re.Pattern, type(None)), + ) + app.connect("config-inited", _config_inited) return { "parallel_read_safe": True, From a897b378e98182d6f6abf5933b93e7c00359b675 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 5 May 2022 15:40:50 -0700 Subject: [PATCH 04/26] Add python_strip_property_prefix config option --- docs/python.rst | 5 +++++ sphinx_immaterial/python_domain_fixes.py | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/python.rst b/docs/python.rst index a4b2398df..bac77cace 100644 --- a/docs/python.rst +++ b/docs/python.rst @@ -128,6 +128,11 @@ Python domain customization Does something with the object. +.. confval:: python_strip_property_prefix + + Strip the ``property`` prefix from :rst:dir:`py:property` object + descriptions. + Overloaded functions -------------------- diff --git a/sphinx_immaterial/python_domain_fixes.py b/sphinx_immaterial/python_domain_fixes.py index bd4302fb2..139b8dbc2 100644 --- a/sphinx_immaterial/python_domain_fixes.py +++ b/sphinx_immaterial/python_domain_fixes.py @@ -12,6 +12,7 @@ Optional, Any, Iterator, + Union, ) import docutils.nodes @@ -151,11 +152,25 @@ def _monkey_patch_python_get_signature_prefix( ) -> None: orig_get_signature_prefix = directive_cls.get_signature_prefix - def get_signature_prefix(self, sig: str): + def get_signature_prefix(self, sig: str) -> Union[str, List[docutils.nodes.Node]]: prefix = orig_get_signature_prefix(self, sig) + if not self.env.config.python_strip_property_prefix: + return prefix if sphinx.version_info >= (4, 3): + prefix = cast(List[docutils.nodes.Node], prefix) + assert isinstance(prefix, list) + for prop_idx, node in enumerate(prefix): + if node == "property": + assert isinstance( + prefix[prop_idx + 1], sphinx.addnodes.desc_sig_space + ) + prefix = list(prefix) + del prefix[prop_idx : prop_idx + 2] + break return prefix - parts = cast(str, prefix).strip().split(" ") + prefix = cast(str, prefix) # type: ignore + assert isinstance(prefix, str) + parts = prefix.strip().split(" ") if "property" in parts: parts.remove("property") if parts: @@ -877,6 +892,9 @@ def setup(app: sphinx.application.Sphinx): rebuild="env", types=(re.Pattern, type(None)), ) + app.add_config_value( + "python_strip_property_prefix", default=False, rebuild="env", types=(bool,) + ) app.connect("config-inited", _config_inited) return { From baac86ad60a7697504b03c0f5df1f91bfb596e26 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Wed, 15 Jun 2022 10:32:31 -0700 Subject: [PATCH 05/26] Disable inlinesyntaxhighlight fix on Sphinx >=5 This fix is available upstream in Sphinx 5.0. --- sphinx_immaterial/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sphinx_immaterial/__init__.py b/sphinx_immaterial/__init__.py index 5b304a17b..2c29eb32d 100644 --- a/sphinx_immaterial/__init__.py +++ b/sphinx_immaterial/__init__.py @@ -17,7 +17,6 @@ from . import apidoc_formatting from . import autodoc_property_type from . import cpp_domain_fixes -from . import inlinesyntaxhighlight from . import generic_synopses from . import nav_adapt from . import object_toc @@ -316,7 +315,10 @@ def setup(app): app.setup_extension(cpp_domain_fixes.__name__) app.setup_extension(nav_adapt.__name__) app.setup_extension(postprocess_html.__name__) - app.setup_extension(inlinesyntaxhighlight.__name__) + if sphinx.version_info < (5, 0): + from . import inlinesyntaxhighlight # pylint: disable=import-outside-toplevel + + app.setup_extension(inlinesyntaxhighlight.__name__) app.setup_extension(object_toc.__name__) app.setup_extension(search_adapt.__name__) app.setup_extension(generic_synopses.__name__) From 330e71f0288b9a4972855afcaf8d98bcad9d4190 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 16 Jun 2022 11:42:38 -0700 Subject: [PATCH 06/26] Fix formatting of content_tabs.py --- sphinx_immaterial/content_tabs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sphinx_immaterial/content_tabs.py b/sphinx_immaterial/content_tabs.py index 92ad19199..9649b310b 100644 --- a/sphinx_immaterial/content_tabs.py +++ b/sphinx_immaterial/content_tabs.py @@ -174,6 +174,7 @@ def visit_tab_set(self: HTMLTranslator, node: content_tab_set): def depart_tab_set(self: HTMLTranslator, node: content_tab_set): pass + def setup(app: Sphinx): app.add_directive("md-tab-set", MaterialTabSetDirective) app.add_directive("md-tab-item", MaterialTabItemDirective) From 5fe3c05fb19b44e2d55676fa18106ce394dc0c3b Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 16 Jun 2022 11:43:19 -0700 Subject: [PATCH 07/26] Support section titles in Python object description content --- sphinx_immaterial/python_domain_fixes.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/sphinx_immaterial/python_domain_fixes.py b/sphinx_immaterial/python_domain_fixes.py index 139b8dbc2..d84b223d9 100644 --- a/sphinx_immaterial/python_domain_fixes.py +++ b/sphinx_immaterial/python_domain_fixes.py @@ -846,6 +846,34 @@ def handle_signature( sphinx.domains.python.PyObject.handle_signature = handle_signature +def _monkey_patch_python_domain_to_support_titles(): + """Enables support for titles in all Python directive types. + + Normally sphinx only supports titles in `automodule`, but the python_apigen + extension uses titles to group member summaries. + """ + + orig_before_content = PyObject.before_content + + def before_content(self: sphinx.domains.python.PyObject) -> None: + orig_before_content(self) + setattr(self, "_saved_content", self.content) + self.content = docutils.statemachine.StringList() + + orig_transform_content = sphinx.domains.python.PyObject.transform_content + + def transform_content(self: PyObject, contentnode: docutils.nodes.Node) -> None: + sphinx.util.nodes.nested_parse_with_titles( + self.state, + getattr(self, "_saved_content"), + contentnode, + ) + orig_transform_content(self, contentnode) + + sphinx.domains.python.PyObject.before_content = before_content + sphinx.domains.python.PyObject.transform_content = transform_content + + def _config_inited( app: sphinx.application.Sphinx, config: sphinx.config.Config ) -> None: @@ -879,6 +907,7 @@ def setup(app: sphinx.application.Sphinx): _monkey_patch_python_domain_to_add_object_synopses_to_references() _monkey_patch_python_domain_to_support_synopses() _monkey_patch_python_domain_to_support_object_ids() + _monkey_patch_python_domain_to_support_titles() sphinx.domains.python.PythonDomain.initial_data["synopses"] = {} # name -> synopsis From a97b57428961675a8dae2aa8a759a080afb8f16a Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Fri, 17 Jun 2022 18:39:50 -0700 Subject: [PATCH 08/26] Fix conf.py to avoid extlinks warning in Sphinx 5.0 --- docs/conf.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e900cd7d4..8c9904039 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -169,10 +169,16 @@ extlinks = { "duref": ( "http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s", - "", + "rST %s", + ), + "durole": ( + "http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s", + "rST role %s", + ), + "dudir": ( + "http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s", + "rST directive %s", ), - "durole": ("http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s", ""), - "dudir": ("http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s", ""), } object_description_options = [] From 7059217ef0bbd28c76540b3adaa2b0fc6329225e Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Tue, 21 Jun 2022 15:16:20 -0700 Subject: [PATCH 09/26] Add hide-edit-link page meta option --- docs/customization.rst | 14 ++++++++++++++ sphinx_immaterial/nav_adapt.py | 19 +++++++++---------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/customization.rst b/docs/customization.rst index 23d3b321e..ab340bd14 100644 --- a/docs/customization.rst +++ b/docs/customization.rst @@ -158,6 +158,20 @@ Each metadata is evaluated as a ``:key: value`` pair. :tocdepth: 0 +.. themeconf:: hide-edit-link + + If specified, hides the "Edit this page" link at the top of the page. By + default, an edit link is shown if :themeconf:`edit_uri` is specified. This + option overrides that for a given page. + + .. code-block:: rst + :caption: Hide the "Edit this page" link: + + :hide-edit-link: + + A common use case for this option is to specify it on automatically-generated + pages, as for those pages there is no source document to edit. + Configuration Options ===================== diff --git a/sphinx_immaterial/nav_adapt.py b/sphinx_immaterial/nav_adapt.py index 6b45f9753..772ba0776 100644 --- a/sphinx_immaterial/nav_adapt.py +++ b/sphinx_immaterial/nav_adapt.py @@ -569,13 +569,13 @@ def _html_page_context( page_title = markupsafe.Markup.escape( markupsafe.Markup(context.get("title")).striptags() ) - meta = context.get("meta", {}) + meta = context.get("meta") + if meta is None: + meta = {} global_toc, local_toc = _get_mkdocs_tocs( app, pagename, - duplicate_local_toc=bool( - meta and isinstance(meta.get("duplicate-local-toc"), str) - ), + duplicate_local_toc=isinstance(meta.get("duplicate-local-toc"), str), ) context.update(nav=_NavContextObject(global_toc)) context["nav"].homepage = dict( @@ -615,11 +615,10 @@ def _html_page_context( meta={"hide": [], "revision_date": context.get("last_updated")}, content=context.get("body"), ) - if meta: - if meta.get("tocdepth") == 0 or "hide-toc" in meta.keys(): - page["meta"]["hide"].append("toc") - if "hide-navigation" in meta.keys(): - page["meta"]["hide"].append("navigation") + if meta.get("tocdepth") == 0 or "hide-toc" in meta: + page["meta"]["hide"].append("toc") + if "hide-navigation" in meta: + page["meta"]["hide"].append("navigation") if context.get("next"): page["next_page"] = { "title": markupsafe.Markup.escape( @@ -636,7 +635,7 @@ def _html_page_context( } repo_url: Optional[str] = theme_options.get("repo_url") edit_uri: Optional[str] = theme_options.get("edit_uri") - if repo_url and edit_uri and not READTHEDOCS: + if repo_url and edit_uri and not READTHEDOCS and "hide-edit-link" not in meta: page["edit_url"] = "/".join( [ repo_url.rstrip("/"), From d08c103d77e1f1f795a88632b6790e2c7d32c479 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Tue, 21 Jun 2022 17:06:18 -0700 Subject: [PATCH 10/26] Fix error in case of empty global TOC Previously, if the global toc was empty, the build failed with an error. With this change, an empty TOC is correctly supported. --- sphinx_immaterial/nav_adapt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx_immaterial/nav_adapt.py b/sphinx_immaterial/nav_adapt.py index 772ba0776..8cb9e022e 100644 --- a/sphinx_immaterial/nav_adapt.py +++ b/sphinx_immaterial/nav_adapt.py @@ -189,7 +189,8 @@ def visit_list_item(self, node: docutils.nodes.list_item): def _get_mkdocs_toc( - toc_node: docutils.nodes.Node, builder: sphinx.builders.html.StandaloneHTMLBuilder + toc_node: Optional[docutils.nodes.Node], + builder: sphinx.builders.html.StandaloneHTMLBuilder, ) -> List[MkdocsNavEntry]: """Converts a docutils toc node into a mkdocs-format JSON toc.""" visitor = _TocVisitor(sphinx.util.docutils.new_document(""), builder) @@ -431,7 +432,6 @@ def __init__(self, app: sphinx.application.Sphinx): maxdepth=-1, titles_only=False, ) - assert global_toc_node is not None global_toc = _get_mkdocs_toc(global_toc_node, builder) _add_domain_info_to_toc(app, global_toc, fake_pagename) self.entries = global_toc From c548006675e3784aca8872b825381f72da4feb05 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Tue, 28 Jun 2022 17:51:34 -0700 Subject: [PATCH 11/26] docs: Support ENUM config options in the confval directive --- docs/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 8c9904039..c7f973724 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,6 +11,8 @@ import sys import typing +from typing_extensions import Literal + sys.path.insert(0, os.path.abspath(".")) import docutils @@ -317,6 +319,8 @@ def _parse_confval_signature( logger.error("Invalid config option: %r", signature) else: default, rebuild, types = registry_option + if isinstance(types, sphinx.config.ENUM): + types = (Literal[tuple(types.candidates)],) if isinstance(types, type): types = (types,) if types: From dc5006a5caa7a110967e530629b2668566236e46 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Tue, 28 Jun 2022 18:06:55 -0700 Subject: [PATCH 12/26] docs: Avoid @functools.cached_property on Python < 3.8 --- docs/test_py_module/test.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/test_py_module/test.py b/docs/test_py_module/test.py index c7e260840..24030c3b7 100644 --- a/docs/test_py_module/test.py +++ b/docs/test_py_module/test.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Test Module for sphinx_rtd_theme.""" import functools +import sys from typing import Union @@ -38,7 +39,7 @@ class Foo: #: It can have multiple lines. bar = 1 - flox = 1.5 #: Doc comment for Foo.flox. One line only. + flox = 1.5 #: Doc comment for Foo.flox. One line only. baz = 2 """Docstring for class attribute Foo.baz.""" @@ -119,10 +120,12 @@ def qux_caps(self) -> str: """Return the instance qux as uppercase.""" return self.capitalize(self.qux) - @functools.cached_property - def qux_caps_cached(self) -> str: - """Return the cached value of instance qux as uppercase.""" - return self.qux_caps + if sys.version_info >= (3, 8): + + @functools.cached_property + def qux_caps_cached(self) -> str: + """Return the cached value of instance qux as uppercase.""" + return self.qux_caps def func(long: int, param: str, args: None, flags: bool, lists: Union[list, tuple]): From 1dbd35191db66c99895ac322f2cdd2e3220328ef Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Wed, 29 Jun 2022 12:40:16 -0700 Subject: [PATCH 13/26] Exclude _sphinx_javascript_frameworks_compat.js by default This JavaScript file is added by the basic theme but is not required by this theme. --- sphinx_immaterial/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sphinx_immaterial/__init__.py b/sphinx_immaterial/__init__.py index 2c29eb32d..ab398d18d 100644 --- a/sphinx_immaterial/__init__.py +++ b/sphinx_immaterial/__init__.py @@ -134,6 +134,7 @@ def init_js_files(self): ) if nav_adapt.READTHEDOCS is None: excluded_scripts.add("_static/jquery.js") + excluded_scripts.add("_static/_sphinx_javascript_frameworks_compat.js") self.script_files = [ x for x in self.script_files if x.filename not in excluded_scripts ] @@ -190,6 +191,9 @@ def onerror(filename: str, error: Exception) -> None: ] if nav_adapt.READTHEDOCS is None: excluded_list.append("**/jquery*.js") + excluded_list.append( + "**/_sphinx_javascript_frameworks_compat.js" + ) excluded = sphinx.util.matching.Matcher(excluded_list) else: excluded = sphinx.util.matching.DOTFILES From 1d793aa8ec78a2ee843a9aeb5f9917e3dfc287b1 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Wed, 29 Jun 2022 13:54:14 -0700 Subject: [PATCH 14/26] Avoid warning in cpp_domain_fixes is setup more than once This can occur in unit tests. --- sphinx_immaterial/cpp_domain_fixes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sphinx_immaterial/cpp_domain_fixes.py b/sphinx_immaterial/cpp_domain_fixes.py index 77ce5ed90..e51433e7c 100644 --- a/sphinx_immaterial/cpp_domain_fixes.py +++ b/sphinx_immaterial/cpp_domain_fixes.py @@ -1071,6 +1071,8 @@ def setup(app: sphinx.application.Sphinx): desc_cpp_requires_clause, desc_cpp_explicit, ): + if node in sphinx.addnodes.SIG_ELEMENTS: + continue app.add_node(node) sphinx.addnodes.SIG_ELEMENTS.append(node) From 8e4485e69364830a4020618899a71623324b284f Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 5 May 2022 15:47:09 -0700 Subject: [PATCH 15/26] Add python_apigen extension Co-authored-by: Brendan <2bndy5@gmail.com> --- .github/workflows/build.yml | 2 + .gitignore | 2 +- .pylintrc | 3 +- dev-requirements.txt | 1 + docs/conf.py | 31 +- docs/customization.rst | 12 + docs/index.rst | 2 + docs/python_apigen.rst | 512 +++++ docs/python_apigen_demo.rst | 12 + docs/tensorstore_demo/__init__.py | 1328 ++++++++++++ docs/tensorstore_demo/_tensorstore.py | 58 + sphinx_immaterial/apidoc_formatting.py | 2 +- sphinx_immaterial/autodoc_property_type.py | 38 +- sphinx_immaterial/python_apigen.py | 1878 +++++++++++++++++ sphinx_immaterial/sphinx_utils.py | 46 +- src/assets/stylesheets/main/layout/_nav.scss | 12 +- tests/python_apigen_test.py | 82 + tests/python_apigen_test_modules/__init__.py | 0 .../alphabetical.py | 6 + .../python_apigen_test_modules/classmethod.py | 11 + tools/build/index.ts | 13 +- 21 files changed, 4030 insertions(+), 21 deletions(-) create mode 100644 docs/python_apigen.rst create mode 100644 docs/python_apigen_demo.rst create mode 100644 docs/tensorstore_demo/__init__.py create mode 100644 docs/tensorstore_demo/_tensorstore.py create mode 100644 sphinx_immaterial/python_apigen.py create mode 100644 tests/python_apigen_test.py create mode 100644 tests/python_apigen_test_modules/__init__.py create mode 100644 tests/python_apigen_test_modules/alphabetical.py create mode 100644 tests/python_apigen_test_modules/classmethod.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 854765595..b432f415a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,6 +55,8 @@ jobs: dist/*.tar.* - name: Install wheel and docs' dependencies run: pip install dist/*.whl -r docs/requirements.txt + - run: pytest -vv -s tests/ + name: Run Python tests - name: Build html docs working-directory: docs run: sphinx-build . _build/html diff --git a/.gitignore b/.gitignore index 98f2629ac..7545ae6e1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ __pycache__/ # Sphinx documentation docs/_build/ -docs/generated/ +docs/python_apigen_generated/ # mypy .mypy_cache/ diff --git a/.pylintrc b/.pylintrc index 36d639858..49eb18307 100644 --- a/.pylintrc +++ b/.pylintrc @@ -111,7 +111,8 @@ disable=raw-checker-failed, too-many-instance-attributes, bad-option-value, too-many-return-statements, - line-too-long + line-too-long, + fixme # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/dev-requirements.txt b/dev-requirements.txt index 7d7a8ea91..caae34e5d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,3 +5,4 @@ mypy types-PyYAML docutils-stubs types-jsonschema +pytest diff --git a/docs/conf.py b/docs/conf.py index c7f973724..82b1a51de 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,6 +45,7 @@ "sphinx.ext.doctest", "sphinx.ext.extlinks", "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.mathjax", "sphinx.ext.viewcode", @@ -54,6 +55,7 @@ "sphinx_immaterial.format_signatures", "sphinx_immaterial.cppreference", "sphinx_immaterial.json_domain", + "sphinx_immaterial.python_apigen", "sphinx_jinja", ] @@ -66,9 +68,6 @@ # documents. default_role = "any" -autosummary_generate = True -autoclass_content = "class" - # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -228,6 +227,10 @@ .. role:: json(code) :language: json :class: highlight + +.. role:: rst(code) + :language: rst + :class: highlight """ @@ -276,6 +279,15 @@ json_schemas = ["index_transform_schema.yml", "inheritance_schema.yml"] +python_apigen_modules = {"tensorstore_demo": "python_apigen_generated/"} + +python_apigen_default_groups = [ + ("class:.*", "Classes"), + (r".*:.*\.__(init|new)__", "Constructors"), + (r".*:.*\.__eq__", "Comparison operators"), + (r".*:.*\.__(str|repr)__", "String representation"), +] + def _validate_parallel_build(app): # Verifies that all of the extensions defined by this theme support parallel @@ -291,7 +303,7 @@ def _parse_object_description_signature( registry_option = registry.get(signature) node += sphinx.addnodes.desc_name(signature, signature) if registry_option is None: - logger.error("Invalid object description option: %r", signature) + logger.error("Invalid object description option: %r", signature, location=node) else: node += sphinx.addnodes.desc_sig_punctuation(" : ", " : ") annotations = sphinx.domains.python._parse_annotation( @@ -316,7 +328,7 @@ def _parse_confval_signature( registry_option = values.get(signature) node += sphinx.addnodes.desc_name(signature, signature) if registry_option is None: - logger.error("Invalid config option: %r", signature) + logger.error("Invalid config option: %r", signature, location=node) else: default, rebuild, types = registry_option if isinstance(types, sphinx.config.ENUM): @@ -365,4 +377,13 @@ def setup(app): indextemplate="pair: %s; object description option", parse_node=_parse_object_description_signature, ) + + # Add `event` type from Sphinx's own documentation, to allow intersphinx + # references to Sphinx events. + app.add_object_type( + "event", + "event", + objname="Sphinx event", + indextemplate="pair: %s; event", + ) app.connect("builder-inited", _validate_parallel_build) diff --git a/docs/customization.rst b/docs/customization.rst index ab340bd14..c572140eb 100644 --- a/docs/customization.rst +++ b/docs/customization.rst @@ -416,11 +416,23 @@ Configuration Options configuration option takes precedence over :themeconf:`toc_title_is_page_title`. + .. code-block:: python + + html_theme_options = { + "toc_title": "Contents", + } + .. themeconf:: toc_title_is_page_title If set to ``True`` and :themeconf:`toc_title` is unspecified, the table of contents is labeled in the side bar by the title of the page itself. + .. code-block:: python + + html_theme_options = { + "toc_title_is_page_title": True, + } + .. themeconf:: version_dropdown A `bool` flag indicating whether the version drop-down selector should be used. See diff --git a/docs/index.rst b/docs/index.rst index 02cb7e244..fef5bac91 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -81,6 +81,7 @@ or ``theme.conf`` for more details. api format_signatures python + python_apigen cpp external_cpp_references cppreference @@ -91,6 +92,7 @@ or ``theme.conf`` for more details. :hidden: demo_api + python_apigen_demo specimen rst_basics rst-cheatsheet/rst-cheatsheet diff --git a/docs/python_apigen.rst b/docs/python_apigen.rst new file mode 100644 index 000000000..de2a3ae76 --- /dev/null +++ b/docs/python_apigen.rst @@ -0,0 +1,512 @@ +Python API documentation generation +=================================== + +This theme includes an optional extension that generates Python API +documentation pages. + +- A separate page is generated for each documented entity. + +- Top-level module-level entities are organized into *groups*, specified by the + ``:group:`` field within the docstring. The :rst:dir:`python-apigen-group` + directive is used to insert a *summary* of a group into an existing page. + +- The summary of a group shows an abbreviated signature and an abbreviated + description for each entity in the group; the name portion of the signature is + a link to the separate page for that entity. + +- Class members are also organized into groups, also using the ``:group:`` field + within their docstrings. Class members without a group are assigned to a + default group based on their name. The summaries for each class member group + are included on the page that documents the class. + +- There is special support for pybind11-defined overloaded functions. Each + overload is documented as a separate function; each overload is identified by + the ``:overload:`` field specified within its docstring. + +Usage +----- + +To use this extension, add :python:`"sphinx_immaterial.python_apigen"` it to the +list of extensions in :file:`conf.py` and define the +:confval:`python_apigen_modules` configuration option. + +For example: + +.. code-block:: python + + extensions = [ + # other extensions... + "sphinx_immaterial.python_apigen", + ] + + python_apigen_modules = { + "my_module": "api", + } + +This configuration is sufficient to generate a documentation page for every +top-level member of the specified modules. To generate a listing of those +top-level members and add the generated documentation pages to the global table +of contents, the :rst:dir:`python-apigen-group` directive may be used. + +.. tip:: + + This extension works well with the :themeconf:`toc_title_is_page_title` + configuration option. + +Groups +^^^^^^ + +Each documented module member and class member is assigned to a member group. +The group is either explicitly specified or determined automatically. + +The group may be specified explicitly using the :rst:`:group:` field within the +docstring: + +.. code-block:: python + + def foo(x: int) -> int: + """Does something or other. + + :param x: The parameter. + :group: My group + +When using the :py:obj:`sphinx.ext.napoleon` extension, the group can also be +specified using the ``Group:`` section: + +.. code-block:: python + + def foo(x: int) -> int: + """Does something or other. + + Args: + x: The parameter. + + Group: + My group + +If the group is not specified explicitly, it is determined automatically based +on the :confval:`python_apigen_default_groups` option. + +.. _python-apigen-member-order: + +Member order +^^^^^^^^^^^^ + +For each each member there is also assigned an associated integer ``order`` +value that determines the order in which members are listed: members are listed +in ascending order by their associated ``order`` value, and ties are broken +according to the :confval:`python_apigen_order_tiebreaker` option. + +Similarly to the group name, the order value may be specified explicitly using +the :rst:`:order:` field within the docstring: + +.. code-block:: python + + def foo(x: int) -> int: + """Does something or other. + + :param x: The parameter. + :group: My group + :order: 1 + +When using the :py:obj:`sphinx.ext.napoleon` extension, the group can also be +specified using the ``Order:`` section: + +.. code-block:: python + + def foo(x: int) -> int: + """Does something or other. + + Args: + x: The parameter. + + Group: + My group + + Order: + 1 + +If the order value is not specified explicitly, it is determined automatically +based on the :confval:`python_apigen_default_order` option. + +The order associated with a member determines both: + +- the relative order in which the member is listed within its associated group; +- for class members, the order of the per-group sections for which an explicit + section is not already listed in the class docstring (the order value + associated with a group is the minimum of the order values of all of its + members). + +rST Directives +^^^^^^^^^^^^^^ + +.. rst:directive:: .. python-apigen-group:: group-name + + Generates a summary of all top-level members that are in the specified group, + and also inserts a table-of-contents entry for each member at the current + document position. + + Before matching the specified group name to the group name of every top-level + member, all the group names are normalized by converting each letter to + lowercase and converting spaces to ``-``. + + .. rst:directive:option:: notoc + + By default, this directive also adds the pages corresponding to the + members of the specified group to the global table of contents as children + of the current page/section. Specifying this flag disables that behavior. + + The group namespace for module-level members is global: if module ``a`` + defines a member ``foo`` in group ``My group`` and module ``b`` defines a + member ``bar`` that is also in ``My group``, then the following example would + insert a summary of both ``a.foo`` and ``b.bar``: + + .. rst-example:: Example usage + + .. python-apigen-group:: Some other group + :notoc: + + .. note:: + + This directive only includes top-level module members (for the modules + specified by :confval:`python_apigen_modules`). Class members are also + organized into groups, but these groups are per-class and are listed + (along with their members) on the documentation page for the class. + +.. rst:directive:: .. python-apigen-entity-summary:: entity-name + + Generates a summary of a single Python entity. + + The ``entity-name`` should be specified as + :python:`module_name.ClassName.member` or + :python:`module_name.ClassName.member(overload)`. + + .. rst:directive:option:: notoc + + By default, this directive also adds the page corresponding to the + specified Python entity to the global table of contents as a child of the + current page/section. Specifying this flag disables that behavior. + + .. rst-example:: Example usage + + .. python-apigen-entity-summary:: tensorstore_demo.IndexDomain.__init__(json) + :notoc: + +Sections defined within docstrings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When using this extension, docstrings can define sections, including nested +sections, using the usual `reStructedText section syntax`. The +mapping between punctuation characters and heading levels is local to the +individual docstring. Therefore, it is not necessary (though still recommended) +to use a consistent order of punctuation characters across different docstrings. + +In addition to providing a way to organize any explanatory content, for classes, +sections can also correspond to member groups, as described below. + +.. _python-apigen-class-member-groups: + +Class member groups +^^^^^^^^^^^^^^^^^^^ + +Within the documentation page for a given class, after its normal docstring +content, a summary is added for each documented member, with a separate section +per group. + +For each group, if the class docstring defines a section with a title equal to +the group name (or an id equal to the normalized group name), the member +summaries are added to the end of the existing section. Otherwise, a new +section for the group is added to the end of the class documentation. + +New sections are added in the order of first ocurrence of the group within the +:ref:`order` defined for the members. + +For example, consider the following class definition: + +.. code-block:: python + + class Foo: + """This is some class. + + Constructors + ------------ + + This class defines the following constructors. + + Operations + ---------- + + This class supports the following operations. + """ + + def __init__(self): + """Constructs the class. + + :group: Consructors + """ + + def foo(self): + """Performs the foo operation. + + :group: Operations + """ + + def bar(self): + """Performs the bar operation. + + :group: Operations + """ + + def size(self) -> int: + """Returns the size. + + :group: Accessors + :order: 2 + """ + + def __setitem__(self, i: int, value: int) -> None: + """Set the element at the given position. + + :group: Indexing + :order: 3 + """ + + def __getitem__(self, i: int) -> int: + """Returns the element at the given position. + + :group: Indexing + :order: 1 + """ + +The ``__init__`` method will be documented within the existing ``Constructors`` +section, the ``foo`` and ``bar`` methods will be documented within the existing +``Operations`` section. After the ``Operations`` section, a new ``Indexing`` +section will be added that lists the ``__getitem__`` and ``__setitem__`` +members, and then a new ``Accessors`` section will be added that lists the +``size`` method. + +Configuration +------------- + +.. confval:: python_apigen_modules + + Maps module names to the output path prefix relative to the source directory. + + All entities defined by the specified modules are documented. + + For example, with the following added to :file:`conf.py`: + + .. code-block:: python + + python_apigen_modules = { + "my_module": "my_api/", + "my_other_module": "other_api/my_other_module.", + } + + The following generated documents will be used (depending on the value of + :confval:`python_apigen_case_insensitive_filesystem`): + + .. jinja:: sys + + {%- set example_python_apigen_modules = { + "my_module": "my_api/", + "my_other_module": "other_api/my_other_module.", + } + %} + {%- set example_python_apigen_objects = [ + ("my_module.foo", ""), + ("my_module.Foo", ""), + ("my_module.Foo.method", ""), + ("my_module.Foo.__init__", "json"), + ("my_module.Foo.__init__", "values"), + ("my_module.Bar", ""), + ("my_other_module.Baz", ""), + ] + %} + {%- set python_apigen_get_docname = sys.modules["sphinx_immaterial.python_apigen"]._get_docname %} + + .. list-table:: + :widths: auto + :header-rows: 1 + + * - Python object + - Overload + - Document (case-sensitive) + - Document (case-insensitive) + + {%- for full_name, overload_id in example_python_apigen_objects %} + * - :python:`{{ full_name }}` + - {{ "``" + overload_id + "``" if overload_id else "" }} + - :file:`{{ python_apigen_get_docname(example_python_apigen_modules, full_name, overload_id, False) }}` + - :file:`{{ python_apigen_get_docname(example_python_apigen_modules, full_name, overload_id, True) }}` + {%- endfor %} + + .. note:: + + The specified path prefix for each module is treated as a *prefix*, not a + directory. It should normally end in either :python:`"/"` or some other + delimiter like :python:`"."` If you want the generated document name to + include the module name, choose a prefix of the form + :python:`"api_directory/module_name."`. If you want the generated + document name to exclude the module name, choose a prefix of the form + :python:`"api_directory/"`. + + .. warning:: + + Because Sphinx is not designed to process files outside the source tree, + these files are actually written to the source tree, and are regenerated + automatically at the start of the build. These files should not be + checked into your source repository. (When using git, you may wish to add + a suitable pattern to a :file:`.gitignore` file.) + + The generated files start with a special comment to indicate that they + were generated by this extension. Stale files from previous build + invocations are deleted automatically. If there is an existing + non-generated file with the same name as a to-be-generated file, the + existing file will not be overwritten and the build will fail (showing an + error message). + +.. confval:: python_apigen_default_groups + + :python:`list` of :python:`(pattern, group)` pairs, where :python:`pattern` + is a regular expression matching strings of the form + :python:`":"` + (e.g. :python:`"method:module_name.ClassName.member"`) and :python:`group` is + the group name to assign. + + The group name for a given member is determined by the *last* matching + pattern. If no pattern matches, the group is ``Public members``. + + .. code-block:: python + :caption: Example addition to :file:`conf.py` + + python_apigen_default_groups = [ + ("class:.*", "Classes"), + (r".*\.__(init|new)__", "Constructors"), + (r".*\.__(str|repr)__", "String representation"), + ] + +.. confval:: python_apigen_default_order + + :python:`list` of :python:`(pattern, order)` pairs, where :python:`pattern` + is a regular expression matching strings of the form + :python:`":"` + (e.g. :python:`"method:module_name.ClassName.member"`) and :python:`order` is + the :py:obj:`int` order to assign. + + The order value for a given member is determined by the *last* matching + pattern. If no pattern matches, the order value is 0. + + .. code-block:: python + :caption: Example addition to :file:`conf.py` + + python_apigen_default_order = [ + ("class:.*", -10), + (r".*\.__(init|new)__", -5), + (r".*\.__(str|repr)__", 5), + ] + +.. confval:: python_apigen_order_tiebreaker + + Specifies the relative order of members that have the same associated + ``order`` value. + + :python:`"definition_order"` + - Top-level members are sorted first by the order their containing module + is listed in :confval:`python_apigen_modules` and then by the order in + which they are defined. + - Class members are sorted by the order in which they are defined. + Inherited members are listed after direct members, according to the + method resolution order. + + :python:`"alphabetical"` + All members are sorted alphabetically, first using case-insensitive + comparison and then breaking ties with case-sensitive comparison. + + .. code-block:: python + :caption: Add to :file:`conf.py` to specify alphabetical order. + + python_apigen_order_tiebreaker = "alphabetical" + +.. confval:: python_apigen_case_insensitive_filesystem + + This extension results in an output file for each documented Python object + based on its fully-qualified name. Python names are case-sensitive, meaning + both :python:`foo` and :python:`Foo` can be defined within the same scope, + but some filesystems are case insensitive (e.g. on Windows and macOS), which + creates the potential for a conflict. + + By default (if :confval:`python_apigen_case_insensitive_filesystem` is + :python:`None`), this extension detects automatically if the filesystem is + case-insensitive, but detection is skipped if the option is set to an + explicit value of :python:`True` or :python:`False`: + + .. code-block:: python + :caption: Add to :file:`conf.py` to force case-insensitive naming scheme + + python_apigen_case_insensitive_filesystem = True + + If the filesystem is either detected or specified to be case-insensitive, + case conflicts are avoided by including a hash in the document name. + +Subscript methods +^^^^^^^^^^^^^^^^^ + +*Subscript methods* are attributes defined on an object that support subscript +syntax. For example: + +.. code-block:: python + + arr.vindex[1, 2:5, [1, 2, 3]] + +These subscript methods can be implemented as follows: + +.. code-block:: python + + class MyArray: + class _Vindex: + def __init__(self, arr: MyArray): + self.arr = arr + + def __getitem__(self, sel: Selection): + # Do something with `self.arr` and `sel`. + return result + + @property + def vindex(self) -> MyArray._Vindex: + return MyArray._Vindex(self) + +Based on the :confval:`python_apigen_subscript_method_types` option, this +extension can recognize this pattern and display :python:`vindex` as: + +.. code-block:: + + vindex[sel: Selection] + +rather than as a normal property. + +.. confval:: python_apigen_subscript_method_types + + Regular expression pattern that matches the return type annotations of + properties that define subscript methods. + + Return type annotations can be specified either as real annotations or in the + textual signature specified as the first line of the docstring. + + The default value matches any name beginning with an underscore, + e.g. :python:`_Vindex` in the example above. + +.. confval:: python_apigen_show_base_classes + + Display the list of base classes when documenting classes. + + Unlike the built-in `sphinx.ext.autodoc` module, base classes are shown using + the normal Python syntax in a parenthesized list after the class name. + + The list of base classes displayed for each class can be customized by adding + a listener to the `autodoc-process-bases` event. This is useful for + excluding base classes that are not intended to be part of the public API. + + .. note:: + + The built-in :py:obj:`object` type is never included as a base class. diff --git a/docs/python_apigen_demo.rst b/docs/python_apigen_demo.rst new file mode 100644 index 000000000..404666a0e --- /dev/null +++ b/docs/python_apigen_demo.rst @@ -0,0 +1,12 @@ +python_apigen demo +================== + +Indexing +-------- + +.. python-apigen-group:: indexing + +Some other group +---------------- + +.. python-apigen-group:: some-other-group diff --git a/docs/tensorstore_demo/__init__.py b/docs/tensorstore_demo/__init__.py new file mode 100644 index 000000000..a18c70be3 --- /dev/null +++ b/docs/tensorstore_demo/__init__.py @@ -0,0 +1,1328 @@ +"""Demo file for Python API documentation generation. + +Some of the docstrings have unusual syntax because they match the output from +pybind11. +""" + +from __future__ import annotations +import typing +import _abc + +from ._tensorstore import * + + +class Dim(): + """1-d index interval with optionally-implicit bounds and dimension label. + + Examples: + + >>> ts.Dim('x') + Dim(label="x") + >>> ts.Dim(inclusive_min=3, exclusive_max=10, label='x') + Dim(inclusive_min=3, exclusive_max=10, label="x") + + See also: + :py:obj:`IndexDomain` + + Group: + Indexing + """ + def __init__(self, *args, **kwargs) -> None: + """__init__(*args, **kwargs) +Overloaded function. + +1. __init__(self: tensorstore_demo.Dim, label: Optional[str] = None, *, implicit_lower: bool = True, implicit_upper: bool = True) -> None + + +Constructs an unbounded interval ``(-inf, +inf)``. + +Args: + label: Dimension label. + implicit_lower: Indicates whether the lower bound is + implicit. + implicit_upper: Indicates whether the upper bound is + implicit. + +Examples: + + >>> x = ts.Dim() + >>> print(x) + (-inf*, +inf*) + >>> x.finite + False + + >>> x = ts.Dim("x", implicit_upper=False) + >>> print(x) + "x": (-inf*, +inf) + >>> x.finite + False + +Overload: + unbounded + + +2. __init__(self: tensorstore_demo.Dim, size: Optional[int], label: Optional[str] = None, *, inclusive_min: Optional[int] = None, implicit_lower: bool = False, implicit_upper: Optional[bool] = None) -> None + + +Constructs a sized interval ``[inclusive_min, inclusive_min+size)``. + +Args: + size: Size of the interval. + label: Dimension label. + inclusive_min: Inclusive lower bound. Defaults to :python:`0`. + implicit_lower: Indicates whether the lower bound is + implicit. + implicit_upper: Indicates whether the upper bound is + implicit. Defaults to :python:`False` if + :python:`size` is specified, otherwise :python:`True`. + +Examples: + + >>> x = ts.Dim(10) + >>> print(x) + [0, 10) + >>> print(ts.Dim(inclusive_min=5, size=10)) + [5, 15) + +Overload: + size + + +3. __init__(self: tensorstore_demo.Dim, inclusive_min: Optional[int] = -inf, exclusive_max: Optional[int] = +inf, *, label: Optional[str] = None, implicit_lower: Optional[bool] = None, implicit_upper: Optional[bool] = None) -> None + + +Constructs a half-open interval ``[inclusive_min, exclusive_max)```. + +Args: + inclusive_min: Inclusive lower bound. + exclusive_max: Exclusive upper bound. + label: Dimension label. + implicit_lower: Indicates whether the lower bound is + implicit. Defaults to :python:`False` if + ``inclusive_min`` is specified, otherwise :python:`True`. + implicit_upper: Indicates whether the upper bound is + implicit. Defaults to :python:`False` if + ``exclusive_max`` is specified, otherwise :python:`True`. + +Examples: + + >>> x = ts.Dim(5, 10) + >>> x + Dim(inclusive_min=5, exclusive_max=10) + >>> print(x) + [5, 10) + +Overload: + exclusive_max + + +4. __init__(self: tensorstore_demo.Dim, *, inclusive_min: Optional[int] = -inf, inclusive_max: Optional[int] = +inf, label: Optional[str] = None, implicit_lower: Optional[bool] = None, implicit_upper: Optional[bool] = None) -> None + + +Constructs a closed interval ``[inclusive_min, inclusive_max]``. + +Args: + inclusive_min: Inclusive lower bound. + inclusive_max: Inclusive upper bound. + label: Dimension label. + implicit_lower: Indicates whether the lower bound is + implicit. Defaults to :python:`False` if + ``inclusive_min`` is specified, otherwise :python:`True`. + implicit_upper: Indicates whether the upper bound is + implicit. Defaults to :python:`False` if + ``exclusive_max`` is specified, otherwise :python:`True`. + +Examples: + + >>> x = ts.Dim(inclusive_min=5, inclusive_max=10) + >>> x + Dim(inclusive_min=5, exclusive_max=11) + >>> print(x) + [5, 11) + +Overload: + inclusive_max +""" + @property + def exclusive_max(self) -> int: + """Exclusive upper bound of the interval. + + Equal to :python:`self.inclusive_max + 1`. If the interval is unbounded above, + equal to the special value of ``+inf+1``. + + Example: + + >>> ts.Dim(inclusive_min=5, inclusive_max=10).exclusive_max + 11 + >>> ts.Dim(exclusive_max=5).exclusive_max + 5 + >>> ts.Dim().exclusive_max + 4611686018427387904 + + Group: + Accessors + + + """ + @property + def inclusive_min(self) -> int: + """Inclusive lower bound of the interval. + + Equal to :python:`self.exclusive_min + 1`. If the interval is unbounded below, + equal to the special value of ``-inf``. + + Example: + + >>> ts.Dim(5).inclusive_min + 0 + >>> ts.Dim(inclusive_min=5, inclusive_max=10).inclusive_min + 5 + >>> ts.Dim().inclusive_min + -4611686018427387903 + + Group: + Accessors + + + """ + @property + def label(self) -> str: + """Dimension label, or the empty string to indicate an unlabeled dimension. + + Example: + + >>> ts.Dim().label + '' + >>> ts.Dim(label='x').label + 'x' + + Group: + Accessors + + + """ + @label.setter + def label(self, arg1: str) -> None: + """Dimension label, or the empty string to indicate an unlabeled dimension. + + Example: + + >>> ts.Dim().label + '' + >>> ts.Dim(label='x').label + 'x' + + Group: + Accessors + """ + @property + def size(self) -> int: + """Size of the interval. + + Equal to :python:`self.exclusive_max - self.inclusive_min`. + + Example: + + >>> ts.Dim(5).size + 5 + >>> ts.Dim(inclusive_min=3, inclusive_max=7).size + 5 + >>> ts.Dim().size + 9223372036854775807 + + Note: + + If the interval is unbounded below or above + (i.e. :python:`self.finite == False`), this value it not particularly + meaningful. + + Group: + Accessors + + + """ + __hash__ = None + pass +class DimExpression(): + """Specifies an advanced indexing operation. + + Dimension expressions permit indexing using + dimension labels, and also support additional operations + that cannot be performed with plain NumPy indexing. + + Group: + Indexing + + Operations + ========== + """ + class _Label(): + def __getitem__(self, *args, **kwargs) -> None: + """__getitem__(self: tensorstore_demo.DimExpression._Label, labels: Union[str, Sequence[str]]) -> tensorstore_demo.DimExpression + + + Sets (or changes) the labels of the selected dimensions. + + Examples: + + >>> ts.IndexTransform(3)[ts.d[0].label['x']] + Rank 3 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) "x" + 1: (-inf*, +inf*) + 2: (-inf*, +inf*) + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + >>> ts.IndexTransform(3)[ts.d[0, 2].label['x', 'z']] + Rank 3 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) "x" + 1: (-inf*, +inf*) + 2: (-inf*, +inf*) "z" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + >>> ts.IndexTransform(3)[ts.d[:].label['x', 'y', 'z']] + Rank 3 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) "x" + 1: (-inf*, +inf*) "y" + 2: (-inf*, +inf*) "z" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + >>> ts.IndexTransform(3)[ts.d[0, 1].label['x', 'y'].translate_by[2]] + Rank 3 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) "x" + 1: (-inf*, +inf*) "y" + 2: (-inf*, +inf*) + Output index maps: + out[0] = -2 + 1 * in[0] + out[1] = -2 + 1 * in[1] + out[2] = 0 + 1 * in[2] + + The new dimension selection is the same as the prior dimension selection. + + Args: + labels: Dimension labels for each selected dimension. + + Returns: + Dimension expression with the label operation added. + + Raises: + IndexError: If the number of labels does not match the number of selected + dimensions, or if the resultant domain would have duplicate labels. + + Group: + Operations + """ + __iter__ = None + pass + class _TranslateTo(): + def __getitem__(self, *args, **kwargs) -> None: + """__getitem__(self: tensorstore_demo.DimExpression._TranslateTo, origins: Union[Sequence[Optional[int]], Optional[int]]) -> tensorstore_demo.DimExpression + + + Translates the domains of the selected input dimensions to the specified + origins without affecting the output range. + + Examples: + + >>> transform = ts.IndexTransform(input_shape=[4, 5, 6], + ... input_labels=['x', 'y', 'z']) + >>> transform[ts.d['x', 'y'].translate_to[10, 20]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [10, 14) "x" + 1: [20, 25) "y" + 2: [0, 6) "z" + Output index maps: + out[0] = -10 + 1 * in[0] + out[1] = -20 + 1 * in[1] + out[2] = 0 + 1 * in[2] + >>> transform[ts.d['x', 'y'].translate_to[10, None]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [10, 14) "x" + 1: [0, 5) "y" + 2: [0, 6) "z" + Output index maps: + out[0] = -10 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + >>> transform[ts.d['x', 'y'].translate_to[10]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [10, 14) "x" + 1: [10, 15) "y" + 2: [0, 6) "z" + Output index maps: + out[0] = -10 + 1 * in[0] + out[1] = -10 + 1 * in[1] + out[2] = 0 + 1 * in[2] + + The new dimension selection is the same as the prior dimension selection. + + Args: + + origins: The new origins for each of the selected dimensions. May also be a + scalar, e.g. :python:`5`, in which case the same origin is used for all + selected dimensions. If :python:`None` is specified for a given dimension, + the origin of that dimension remains unchanged. + + Returns: + Dimension expression with the translation operation added. + + Raises: + + IndexError: + If the number origins does not match the number of selected dimensions. + + IndexError: + If any of the selected dimensions has a lower bound of :python:`-inf`. + + Group: + Operations + """ + __iter__ = None + pass + class _Vindex(): + def __getitem__(self, *args, **kwargs) -> None: + """__getitem__(self: tensorstore_demo.DimExpression._Vindex, indices: typing.Any) -> tensorstore_demo.DimExpression + + + Applies a NumPy-style indexing operation with vectorized indexing semantics. + + This is similar to :py:obj:`DimExpression.__getitem__`, but differs in that if + :python:`indices` specifies any array indexing terms, the broadcasted array + dimensions are unconditionally added as the first dimensions of the result + domain: + + Examples: + + >>> transform = ts.IndexTransform(input_labels=['x', 'y', 'z']) + >>> transform[ts.d['y', 'z'].vindex[[1, 2, 3], [4, 5, 6]]] + Rank 2 -> 3 index space transform: + Input domain: + 0: [0, 3) + 1: (-inf*, +inf*) "x" + Output index maps: + out[0] = 0 + 1 * in[1] + out[1] = 0 + 1 * bounded((-inf, +inf), array(in)), where array = + {{1}, {2}, {3}} + out[2] = 0 + 1 * bounded((-inf, +inf), array(in)), where array = + {{4}, {5}, {6}} + + Returns: + Dimension expression with the indexing operation added. + + Group: + Operations + """ + __iter__ = None + pass + def __getitem__(self, *args, **kwargs) -> None: + """__getitem__(self: tensorstore_demo.DimExpression, indices: typing.Any) -> tensorstore_demo.DimExpression + + + Applies a NumPy-style indexing operation with default index array semantics. + + When using NumPy-style indexing with a dimension expression, all selected + dimensions must be consumed by a term of the indexing spec; there is no implicit + addition of an `Ellipsis` term to consume any remaining dimensions. + + Returns: + Dimension expression with the indexing operation added. + + Group: + Operations + + Examples + ======== + + Integer indexing + ---------------- + + >>> transform = ts.IndexTransform(input_labels=['x', 'y', 'z']) + >>> transform[ts.d['x'][5]] + Rank 2 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) "y" + 1: (-inf*, +inf*) "z" + Output index maps: + out[0] = 5 + out[1] = 0 + 1 * in[0] + out[2] = 0 + 1 * in[1] + >>> transform[ts.d['x', 'z'][5, 6]] + Rank 1 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) "y" + Output index maps: + out[0] = 5 + out[1] = 0 + 1 * in[0] + out[2] = 6 + + A single scalar index term applies to all selected dimensions: + + >>> transform[ts.d['x', 'y'][5]] + Rank 1 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) "z" + Output index maps: + out[0] = 5 + out[1] = 5 + out[2] = 0 + 1 * in[0] + + Interval indexing + ----------------- + + >>> transform = ts.IndexTransform(input_labels=['x', 'y', 'z']) + >>> transform[ts.d['x'][5:10]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [5, 10) "x" + 1: (-inf*, +inf*) "y" + 2: (-inf*, +inf*) "z" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + >>> transform[ts.d['x', 'z'][5:10, 20:30]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [5, 10) "x" + 1: (-inf*, +inf*) "y" + 2: [20, 30) "z" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + + As an extension, TensorStore allows the ``start``, ``stop``, and ``step`` + :py:obj:`python:slice` terms to be vectors rather than scalars: + + >>> transform[ts.d['x', 'z'][[5, 20]:[10, 30]]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [5, 10) "x" + 1: (-inf*, +inf*) "y" + 2: [20, 30) "z" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + >>> transform[ts.d['x', 'z'][[5, 20]:30]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [5, 30) "x" + 1: (-inf*, +inf*) "y" + 2: [20, 30) "z" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + + As with integer indexing, a single scalar slice applies to all selected + dimensions: + + >>> transform[ts.d['x', 'z'][5:30]] + Rank 3 -> 3 index space transform: + Input domain: + 0: [5, 30) "x" + 1: (-inf*, +inf*) "y" + 2: [5, 30) "z" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + + Adding singleton dimensions + --------------------------- + + Specifying a value of ``newaxis`` (equal to `None`) adds a new + dummy/singleton dimension with implicit bounds + :math:`[0, 1)`: + + >>> transform = ts.IndexTransform(input_labels=['x', 'y']) + >>> transform[ts.d[1][ts.newaxis]] + Rank 3 -> 2 index space transform: + Input domain: + 0: (-inf*, +inf*) "x" + 1: [0*, 1*) + 2: (-inf*, +inf*) "y" + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[2] + >>> transform[ts.d[0, -1][ts.newaxis, ts.newaxis]] + Rank 4 -> 2 index space transform: + Input domain: + 0: [0*, 1*) + 1: (-inf*, +inf*) "x" + 2: (-inf*, +inf*) "y" + 3: [0*, 1*) + Output index maps: + out[0] = 0 + 1 * in[1] + out[1] = 0 + 1 * in[2] + + As with integer indexing, if only a single :python:`ts.newaxis` term is + specified, it applies to all selected dimensions: + + >>> transform[ts.d[0, -1][ts.newaxis]] + Rank 4 -> 2 index space transform: + Input domain: + 0: [0*, 1*) + 1: (-inf*, +inf*) "x" + 2: (-inf*, +inf*) "y" + 3: [0*, 1*) + Output index maps: + out[0] = 0 + 1 * in[1] + out[1] = 0 + 1 * in[2] + + ``newaxis`` terms are only permitted in the first operation of a + dimension expression, since in subsequent operations all dimensions of the + dimension selection necessarily refer to existing dimensions: + + .. admonition:: Error + :class: failure + + >>> transform[ts.d[0, 1].translate_by[5][ts.newaxis]] + Traceback (most recent call last): + ... + IndexError: tensorstore_demo.newaxis (`None`) not valid in chained indexing operations + + It is also an error to use ``newaxis`` with dimensions specified by + label: + + .. admonition:: Error + :class: failure + + >>> transform[ts.d['x'][ts.newaxis]] + Traceback (most recent call last): + ... + IndexError: New dimensions cannot be specified by label + + Ellipsis + -------- + + Specifying the special `Ellipsis` value (:python:`...`) is equivalent to + specifying as many full slices :python:`:` as needed to consume the remaining + selected dimensions not consumed by other indexing terms: + + >>> transform = ts.IndexTransform(input_rank=4) + >>> transform[ts.d[:][1, ..., 5].translate_by[3]] + Rank 2 -> 4 index space transform: + Input domain: + 0: (-inf*, +inf*) + 1: (-inf*, +inf*) + Output index maps: + out[0] = 1 + out[1] = -3 + 1 * in[0] + out[2] = -3 + 1 * in[1] + out[3] = 5 + + An indexing spec consisting solely of an `Ellipsis` term has no effect: + + >>> transform[ts.d[:][...]] + Rank 4 -> 4 index space transform: + Input domain: + 0: (-inf*, +inf*) + 1: (-inf*, +inf*) + 2: (-inf*, +inf*) + 3: (-inf*, +inf*) + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[2] + out[3] = 0 + 1 * in[3] + + Integer array indexing + ---------------------- + + Specifying an ``array_like`` *index array* of integer values selects the + coordinates given by the elements of the array of the selected dimension: + + >>> x = ts.array([[1, 2, 3], [4, 5, 6]], dtype=ts.int32) + >>> x = x[ts.d[:].label['x', 'y']] + >>> x[ts.d['y'][[1, 1, 0]]] + TensorStore({ + 'array': [[2, 2, 1], [5, 5, 4]], + 'context': {'data_copy_concurrency': {}}, + 'driver': 'array', + 'dtype': 'int32', + 'transform': { + 'input_exclusive_max': [2, 3], + 'input_inclusive_min': [0, 0], + 'input_labels': ['x', ''], + }, + }) + + As in the example above, if only a single + index array term is specified, the dimensions of the index array are added to + the result domain in place of the selected dimension, consistent with + direct NumPy-style indexing in the default + index array mode. + + However, when using NumPy-style indexing with a dimension expression, if more + than one index array term is specified, the broadcast dimensions of the index + arrays are always added to the beginning of the result domain, i.e. exactly the + behavior of :py:obj:`DimExpression.vindex`. Unlike with direct NumPy-style + indexing (not with a dimension expression), the behavior does not depend on + whether the index array terms apply to consecutive dimensions, since consecutive + dimensions are not well-defined for dimension expressions: + + >>> x = ts.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], dtype=ts.int32) + >>> x = x[ts.d[:].label['x', 'y', 'z']] + >>> x[ts.d['z', 'y'][[1, 0], [1, 1]]] + TensorStore({ + 'array': [[4, 3], [8, 7]], + 'context': {'data_copy_concurrency': {}}, + 'driver': 'array', + 'dtype': 'int32', + 'transform': { + 'input_exclusive_max': [2, 2], + 'input_inclusive_min': [0, 0], + 'input_labels': ['x', ''], + }, + }) + + Boolean array indexing + ---------------------- + + Specifying an ``array_like`` of `bool` values is equivalent to + specifying a sequence of integer index arrays containing the + coordinates of `True` values (in C order), e.g. as obtained from + ``numpy.nonzero``: + + Specifying a 1-d `bool` array is equivalent to a single index array of the + non-zero coordinates: + + >>> x = ts.array([[1, 2, 3], [4, 5, 6]], dtype=ts.int32) + >>> x = x[ts.d[:].label['x', 'y']] + >>> x[ts.d['y'][[False, True, True]]] + TensorStore({ + 'array': [[2, 3], [5, 6]], + 'context': {'data_copy_concurrency': {}}, + 'driver': 'array', + 'dtype': 'int32', + 'transform': { + 'input_exclusive_max': [2, 2], + 'input_inclusive_min': [0, 0], + 'input_labels': ['x', ''], + }, + }) + + Equivalently, using an index array: + + >>> x[ts.d['y'][[1, 2]]] + TensorStore({ + 'array': [[2, 3], [5, 6]], + 'context': {'data_copy_concurrency': {}}, + 'driver': 'array', + 'dtype': 'int32', + 'transform': { + 'input_exclusive_max': [2, 2], + 'input_inclusive_min': [0, 0], + 'input_labels': ['x', ''], + }, + }) + + More generally, specifying an ``n``-dimensional `bool` array is equivalent to + specifying ``n`` 1-dimensional index arrays, where the ``i``\ th index array specifies + the ``i``\ th coordinate of the `True` values: + + >>> x = ts.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]], + ... dtype=ts.int32) + >>> x = x[ts.d[:].label['x', 'y', 'z']] + >>> x[ts.d['x', 'z'][[[True, False, False], [True, True, False]]]] + TensorStore({ + 'array': [[1, 4], [7, 10], [8, 11]], + 'context': {'data_copy_concurrency': {}}, + 'driver': 'array', + 'dtype': 'int32', + 'transform': { + 'input_exclusive_max': [3, 2], + 'input_inclusive_min': [0, 0], + 'input_labels': ['', 'y'], + }, + }) + + Equivalently, using an index array: + + >>> x[ts.d['x', 'z'][[0, 1, 1], [0, 0, 1]]] + TensorStore({ + 'array': [[1, 4], [7, 10], [8, 11]], + 'context': {'data_copy_concurrency': {}}, + 'driver': 'array', + 'dtype': 'int32', + 'transform': { + 'input_exclusive_max': [3, 2], + 'input_inclusive_min': [0, 0], + 'input_labels': ['', 'y'], + }, + }) + + Note that as with integer array indexing, when using NumPy-styling indexing with + a dimension expression, if boolean arrays are applied to more than one selected + dimension, the added dimension corresponding to the `True` values is always + added to the beginning of the result domain, i.e. exactly the behavior of + :py:obj:`DimExpression.vindex`. + + """ + @property + def diagonal(self) -> DimExpression: + """Extracts the diagonal of the selected dimensions. + + The selection dimensions are removed from the resultant index space, and a new + dimension corresponding to the diagonal is added as the first dimension, with an + input domain equal to the intersection of the input domains of the selection + dimensions. The new dimension selection is equal to :python:`ts.d[0]`, + corresponding to the newly added diagonal dimension. + + The lower and upper bounds of the new diagonal dimension are + implicit>> transform = ts.IndexTransform(input_shape=[2, 3], + ... input_labels=["x", "y"]) + >>> transform[ts.d['x', 'y'].diagonal] + Rank 1 -> 2 index space transform: + Input domain: + 0: [0, 2) + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[0] + >>> transform = ts.IndexTransform(3) + >>> transform[ts.d[0, 2].diagonal] + Rank 2 -> 3 index space transform: + Input domain: + 0: (-inf*, +inf*) + 1: (-inf*, +inf*) + Output index maps: + out[0] = 0 + 1 * in[0] + out[1] = 0 + 1 * in[1] + out[2] = 0 + 1 * in[0] + + Note: + + If zero dimensions are selected, :py:obj:`.diagonal` simply results in a new singleton + dimension as the first dimension, equivalent to :python:`[ts.newaxis]`: + + >>> transform = ts.IndexTransform(1) + >>> transform[ts.d[()].diagonal] + Rank 2 -> 1 index space transform: + Input domain: + 0: (-inf*, +inf*) + 1: (-inf*, +inf*) + Output index maps: + out[0] = 0 + 1 * in[1] + + If only one dimension is selected, :py:obj:`.diagonal` is equivalent to + :python:`.label[''].transpose[0]`: + + >>> transform = ts.IndexTransform(input_labels=['x', 'y']) + >>> transform[ts.d[1].diagonal] + Rank 2 -> 2 index space transform: + Input domain: + 0: (-inf*, +inf*) + 1: (-inf*, +inf*) "x" + Output index maps: + out[0] = 0 + 1 * in[1] + out[1] = 0 + 1 * in[0] + + Group: + Operations + + """ + @property + def label(self) -> DimExpression._Label: + pass + @property + def translate_to(self) -> DimExpression._TranslateTo: + pass + @property + def vindex(self) -> DimExpression._Vindex: + pass + __iter__ = None + pass +class IndexDomain(): + """Domain (including bounds and optional dimension labels) of an N-dimensional index space. + + Logically, an :py:class:`.IndexDomain` is the cartesian product of a sequence of :py:obj:`.Dim` objects. + + Note: + + Index domains are immutable, but dimension expressions may be applied + using :py:obj:`.__getitem__(expr)` to obtain a modified domain. + + Group: + Indexing + + """ + def __init__(self, *args, **kwargs) -> None: + """__init__(*args, **kwargs) +Overloaded function. + +1. __init__(self: tensorstore_demo.IndexDomain, rank: Optional[int] = None, *, inclusive_min: Optional[Sequence[int]] = None, implicit_lower_bounds: Optional[Sequence[bool]] = None, exclusive_max: Optional[Sequence[int]] = None, inclusive_max: Optional[Sequence[int]] = None, shape: Optional[Sequence[int]] = None, implicit_upper_bounds: Optional[Sequence[bool]] = None, labels: Optional[Sequence[Optional[str]]] = None) -> None + + +Constructs an index domain from component vectors. + +Args: + rank: Number of dimensions. Only required if no other parameter is specified. + inclusive_min: Inclusive lower bounds for each dimension. If not specified, + defaults to all zero if ``shape`` is specified, otherwise unbounded. + implicit_lower_bounds: Indicates whether each lower bound is + implicit or explicit. Defaults to all explicit if + ``inclusive_min`` or ``shape`` is specified, otherwise defaults to all + implicit. + exclusive_max: Exclusive upper bounds for each dimension. At most one of + ``exclusive_max``, ``inclusive_max``, and ``shape`` may be specified. + inclusive_max: Inclusive upper bounds for each dimension. + shape: Size for each dimension. + implicit_upper_bounds: Indicates whether each upper bound is + implicit or explicit. Defaults to all explicit if + ``exclusive_max``, ``inclusive_max``, or ``shape`` is specified, otherwise + defaults to all implicit. + labels: Dimension labels. Defaults to all unlabeled. + +Examples: + + >>> ts.IndexDomain(rank=5) + { (-inf*, +inf*), (-inf*, +inf*), (-inf*, +inf*), (-inf*, +inf*), (-inf*, +inf*) } + >>> ts.IndexDomain(shape=[2, 3]) + { [0, 2), [0, 3) } + +Overload: + components + + +2. __init__(self: tensorstore_demo.IndexDomain, dimensions: Sequence[tensorstore_demo.Dim]) -> None + + +Constructs an index domain from a :py:class`.Dim` sequence. + +Args: + dimensions: :py:obj:`Sequence` of :py:class`.Dim` objects. + +Examples: + + >>> ts.IndexDomain([ts.Dim(5), ts.Dim(6, label='y')]) + { [0, 5), "y": [0, 6) } + +Overload: + dimensions + + +3. __init__(self: tensorstore_demo.IndexDomain, *, json: Any) -> None + + +Constructs an index domain from its JSON representation. + +Examples: + + >>> ts.IndexDomain( + ... json={ + ... "inclusive_min": ["-inf", 7, ["-inf"], [8]], + ... "exclusive_max": ["+inf", 10, ["+inf"], [17]], + ... "labels": ["x", "y", "z", ""] + ... }) + { "x": (-inf, +inf), "y": [7, 10), "z": (-inf*, +inf*), [8*, 17*) } + +Overload: + json +""" + def __eq__(self, *args, **kwargs) -> None: + """__eq__(self: tensorstore_demo.IndexDomain, arg0: tensorstore_demo.IndexDomain) -> bool + """ + def __getitem__(self, *args, **kwargs) -> None: + """__getitem__(*args, **kwargs) +Overloaded function. + +1. __getitem__(self: tensorstore_demo.IndexDomain, identifier: Union[int, str]) -> tensorstore_demo.Dim + + +Returns the single dimension specified by :python:`identifier`. + +Args: + identifier: Specifies a dimension by integer index or label. As with + :py:obj:`python:list`, a negative index specifies a dimension starting + from the last dimension. + +Examples: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], + ... exclusive_max=[4, 5, 6], + ... labels=['x', 'y', 'z']) + >>> domain[0] + Dim(inclusive_min=1, exclusive_max=4, label="x") + >>> domain['y'] + Dim(inclusive_min=2, exclusive_max=5, label="y") + >>> domain[-1] + Dim(inclusive_min=3, exclusive_max=6, label="z") + +Overload: + identifier + +Group: + Sequence accessors + + +2. __getitem__(self: tensorstore_demo.IndexDomain, selection: typing.Any) -> tensorstore_demo.IndexDomain + + +Returns a new domain with a subset of the dimensions. + +Args: + + selection: Specifies the dimensions to include, either by index or label. May + be any value or sequence of values convertible to a + dimension selection. + +Raises: + ValueError: If any dimension is specified more than once. + +Examples: + + >>> a = ts.IndexDomain(inclusive_min=[1, 2, 3], + ... exclusive_max=[4, 5, 6], + ... labels=['x', 'y', 'z']) + >>> a[:2] + { "x": [1, 4), "y": [2, 5) } + >>> a[0, -1] + { "x": [1, 4), "z": [3, 6) } + >>> a['y', 'x'] + { "y": [2, 5), "x": [1, 4) } + >>> a['y', 1] + Traceback (most recent call last): + ... + ValueError: Input dimensions {1} specified more than once + +Overload: + selection + +Group: + Indexing + + +3. __getitem__(self: tensorstore_demo.IndexDomain, other: tensorstore_demo.IndexDomain) -> tensorstore_demo.IndexDomain + + +Slices this domain by another domain. + +The result is determined by matching dimensions of :python:`other` to dimensions +of :python:`self` either by label or by index, according to one of the following +three cases: + +.. list-table:: + :widths: auto + + * - :python:`other` is entirely unlabeled + + - Result is + :python:`self[ts.d[:][other.inclusive_min:other.exclusive_max]`. + It is an error if :python:`self.rank != other.rank`. + + * - :python:`self` is entirely unlabeled + + - Result is + :python:`self[ts.d[:][other.inclusive_min:other.exclusive_max].labels[other.labels]`. + It is an error if :python:`self.rank != other.rank`. + + * - Both :python:`self` and :python:`other` have at least one labeled dimension. + + - Result is + :python:`self[ts.d[dims][other.inclusive_min:other.exclusive_max]`, where + the sequence of :python:`other.rank` dimension identifiers :python:`dims` + is determined as follows: + + 1. If :python:`other.labels[i]` is specified (i.e. non-empty), + :python:`dims[i] = self.labels.index(other.labels[i])`. It is an + error if no such dimension exists. + + 2. Otherwise, ``i`` is the ``j``\ th unlabeled dimension of :python:`other` + (left to right), and :python:`dims[i] = k`, where ``k`` is the ``j``\ th + unlabeled dimension of :python:`self` (left to right). It is an error + if no such dimension exists. + + If any dimensions of :python:`other` are unlabeled, then it is an error + if :python:`self.rank != other.rank`. This condition is not strictly + necessary but serves to avoid a discrepancy in behavior with normal + domain alignment. + +.. admonition:: Example with all unlabeled dimensions + :class: example + + >>> a = ts.IndexDomain(inclusive_min=[0, 1], exclusive_max=[5, 7]) + >>> b = ts.IndexDomain(inclusive_min=[2, 3], exclusive_max=[4, 6]) + >>> a[b] + { [2, 4), [3, 6) } + +.. admonition:: Example with fully labeled dimensions + :class: example + + >>> a = ts.IndexDomain(inclusive_min=[0, 1, 2], + ... exclusive_max=[5, 7, 8], + ... labels=["x", "y", "z"]) + >>> b = ts.IndexDomain(inclusive_min=[2, 3], + ... exclusive_max=[6, 4], + ... labels=["y", "x"]) + >>> a[b] + { "x": [3, 4), "y": [2, 6), "z": [2, 8) } + +.. admonition:: Example with mixed labeled and unlabeled dimensions + :class: example + + >>> a = ts.IndexDomain(inclusive_min=[0, 0, 0, 0], + ... exclusive_max=[10, 10, 10, 10], + ... labels=["x", "", "", "y"]) + >>> b = ts.IndexDomain(inclusive_min=[1, 2, 3, 4], + ... exclusive_max=[6, 7, 8, 9], + ... labels=["y", "", "x", ""]) + >>> a[b] + { "x": [3, 8), [2, 7), [4, 9), "y": [1, 6) } + +Note: + + On :python:`other`, implicit bounds indicators have no effect. + +Overload: + domain + +Group: + Indexing + + +4. __getitem__(self: tensorstore_demo.IndexDomain, expr: tensorstore_demo.DimExpression) -> tensorstore_demo.IndexDomain + + +Transforms the domain by a dimension expression. + +Args: + expr: Dimension expression to apply. + +Examples: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], + ... exclusive_max=[6, 7, 8], + ... labels=['x', 'y', 'z']) + >>> domain[ts.d[:].translate_by[5]] + { "x": [6, 11), "y": [7, 12), "z": [8, 13) } + >>> domain[ts.d['y'][3:5]] + { "x": [1, 6), "y": [3, 5), "z": [3, 8) } + >>> domain[ts.d['z'][5]] + { "x": [1, 6), "y": [2, 7) } + +Note: + + For the purpose of applying a dimension expression, an + :py:class:`IndexDomain` behaves like an IndexTransform with an + output rank of 0. Consequently, operations that primarily affect the output + index mappings, like integer array indexing, are not very useful, though they + are still permitted. + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], + ... exclusive_max=[6, 7, 8], + ... labels=['x', 'y', 'z']) + >>> domain[ts.d['z'][[3, 5, 7]]] + { "x": [1, 6), "y": [2, 7), [0, 3) } + +Overload: + expr + +Group: + Indexing + + +5. __getitem__(self: tensorstore_demo.IndexDomain, transform: Any) -> tensorstore_demo.IndexDomain + + +Transforms the domain using an explicit index transform. + +Example: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], + ... exclusive_max=[6, 7, 8]) + >>> transform = ts.IndexTransform( + ... input_rank=4, + ... output=[ + ... ts.OutputIndexMap(offset=5, input_dimension=3), + ... ts.OutputIndexMap(offset=-7, input_dimension=0), + ... ts.OutputIndexMap(offset=3, input_dimension=1), + ... ]) + >>> domain[transform] + { [9, 14), [0, 5), (-inf*, +inf*), [-4, 1) } + +Args: + + transform: Index transform, :python:`transform.output_rank` must equal + :python:`self.rank`. + +Returns: + + New domain of rank :python:`transform.input_rank`. + +Note: + + This is equivalent to composing an identity transform over :python:`self` + with :py:param:`.transform`, + i.e. :python:`ts.IndexTransform(self)[transform].domain`. Consequently, + operations that primarily affect the output index mappings, like integer + array indexing, are not very useful, though they are still permitted. + +Overload: + transform + +Group: + Indexing + + """ + def to_json(self, *args, **kwargs) -> None: + """to_json(self: tensorstore_demo.IndexDomain) -> Any + + + Returns the JSON representation. + + Group: + Accessors + """ + @property + def exclusive_max(self) -> typing.Tuple[int, ...]: + """Exclusive upper bound of the domain. + + Example: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], shape=[3, 4, 5]) + >>> domain.exclusive_max + (4, 6, 8) + + Group: + Accessors + + + """ + @property + def inclusive_min(self) -> typing.Tuple[int, ...]: + """Inclusive lower bound of the domain, alias of :py:obj:`.origin`. + + Example: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], shape=[3, 4, 5]) + >>> domain.inclusive_min + (1, 2, 3) + + Group: + Accessors + + + """ + @property + def labels(self) -> typing.Tuple[str, ...]: + """Dimension labels for each dimension. + + Example: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], shape=[3, 4, 5]) + >>> domain.labels + ('', '', '') + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], + ... shape=[3, 4, 5], + ... labels=['x', 'y', 'z']) + >>> domain.labels + ('x', 'y', 'z') + + Group: + Accessors + + + """ + @property + def origin(self) -> typing.Tuple[int, ...]: + """Inclusive lower bound of the domain. + + Example: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], shape=[3, 4, 5]) + >>> domain.origin + (1, 2, 3) + + Group: + Accessors + + + """ + @property + def rank(self) -> int: + """Number of dimensions in the index space. + + Example: + + >>> domain = ts.IndexDomain(shape=[100, 200, 300]) + >>> domain.rank + 3 + + Group: + Accessors + + + """ + @property + def shape(self) -> typing.Tuple[int, ...]: + """Shape of the domain. + + Example: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], shape=[3, 4, 5]) + >>> domain.shape + (3, 4, 5) + + Group: + Accessors + + + """ + @property + def size(self) -> int: + """Total number of elements in the domain. + + This is simply the product of the extents in :py:obj:`.shape`. + + Example: + + >>> domain = ts.IndexDomain(inclusive_min=[1, 2, 3], shape=[3, 4, 5]) + >>> domain.size + 60 + + Group: + Accessors + + + """ + __hash__ = None + pass +class VeryLongClassNameForTestingOutWordWrapping: + """This is a class with a very long name. + + Group: + Some other group + """ + + def very_long_method_name_for_testing_out_word_wrapper(self, x: int) -> int: + """This is a method with a very long name.""" + +class DimensionSelection(DimExpression): + """This extends :py:obj:`DimExpression`. + + Group: + Indexing + """ + + def foo(self, bar: int) -> None: + """Non-inherited method.""" + pass diff --git a/docs/tensorstore_demo/_tensorstore.py b/docs/tensorstore_demo/_tensorstore.py new file mode 100644 index 000000000..f9f37f5e5 --- /dev/null +++ b/docs/tensorstore_demo/_tensorstore.py @@ -0,0 +1,58 @@ +"""Demo file for Python API documentation generation. + +This defines some entities that are aliased into the `tensorstore_demo` module. +""" + +import typing + + +class Foo: + """ + This is a class defined in the ``tensorstore_demo._tensorstore`` module + but should appear to be defined in the ``tensorstore_demo`` module. + + Group: + Some other group + """ + + def bar(self, x: int) -> int: + """Returns :py:param:`x`. + + :param x: Parameter to :py:obj:`.bar`. + """ + pass + + +def bar(x: Foo) -> Foo: + """Returns :py:param:`x`. + + :param x: Parameter to :py:obj:`.bar`. + + Group: + Some other group. + """ + pass + + +class FooSubclass(Foo): + """This is a subclass of :py:obj:`.Foo`. + + Group: + Some other group. + """ + + def baz(self, x: str) -> str: + """Does the baz operation.""" + return x + + @typing.overload + def overloaded_method(self, x: int) -> int: + pass + + @typing.overload + def overloaded_method(self, x: str) -> str: + pass + + def overloaded_method(self, x): + """Returns its argument.""" + return x diff --git a/sphinx_immaterial/apidoc_formatting.py b/sphinx_immaterial/apidoc_formatting.py index 37ff907d2..1dfdbdbb1 100644 --- a/sphinx_immaterial/apidoc_formatting.py +++ b/sphinx_immaterial/apidoc_formatting.py @@ -447,7 +447,7 @@ def setup(app: sphinx.application.Sphinx): # to apply. sphinx.addnodes.desc_signature.classes.append("highlight") - app.connect("object-description-transform", _wrap_signatures) + app.connect("object-description-transform", _wrap_signatures, priority=1000) _monkey_patch_object_description_to_include_fields_in_toc() add_object_description_option( diff --git a/sphinx_immaterial/autodoc_property_type.py b/sphinx_immaterial/autodoc_property_type.py index fe8426bb8..15931be8b 100644 --- a/sphinx_immaterial/autodoc_property_type.py +++ b/sphinx_immaterial/autodoc_property_type.py @@ -1,12 +1,14 @@ """Adds support for obtaining property types from docstring signatures, and improves formatting by PyProperty of type annotations.""" import re -from typing import Tuple, Optional +from typing import Tuple, Optional, Any import sphinx.addnodes import sphinx.domains import sphinx.domains.python import sphinx.ext.autodoc +import sphinx.util.inspect +import sphinx.util.typing PropertyDocumenter = sphinx.ext.autodoc.PropertyDocumenter @@ -14,10 +16,17 @@ property_sig_re = re.compile("^(\\(.*)\\)\\s*->\\s*(.*)$") -def _get_property_return_type(obj: property) -> Optional[str]: - fget = getattr(obj, "fget", None) - if fget is not None: - obj = fget +def _get_property_getter(obj: Any) -> Any: + # property + func = sphinx.util.inspect.safe_getattr(obj, "fget", None) + if func is None: + # cached_property + func = sphinx.util.inspect.safe_getattr(obj, "func", None) + return func + + +def _get_property_return_type(obj: Any) -> Optional[str]: + fget = _get_property_getter(obj) doc = obj.__doc__ if doc is None: return None @@ -40,6 +49,25 @@ def import_object(self: PropertyDocumenter, raiseerror: bool = False) -> bool: result = orig_import_object(self, raiseerror) if not result: return False + + func = _get_property_getter(self.object) + + if func is not None: + try: + signature = sphinx.util.inspect.signature( + func, type_aliases=self.config.autodoc_type_aliases + ) + if ( + signature.return_annotation + is not sphinx.util.inspect.Parameter.empty + ): + self.retann = sphinx.util.typing.stringify( + signature.return_annotation + ) + return True + except TypeError: + pass + if not self.retann: self.retann = _get_property_return_type(self.object) # type: ignore return True diff --git a/sphinx_immaterial/python_apigen.py b/sphinx_immaterial/python_apigen.py new file mode 100644 index 000000000..6a51d32ea --- /dev/null +++ b/sphinx_immaterial/python_apigen.py @@ -0,0 +1,1878 @@ +"""Python API documentation generation extension. + +A separate page is generated for each class/function/member/constant to be +documented. + +As with sphinx.ext.autosummary, we have to physically write a separate rST file +to the source tree for each object to document, as an initial preprocesing step, +since that provides the simplest way to get Sphinx to process those pages. It +is recommended to run the build with the source tree copied to a temporary +directory in order to avoid modifying the real source tree. + +Unlike the sphinx.ext.autosummary extension, we use Sphinx Python domain +directives for the "summaries" as well, rather than a plain table, in order to +display the signatures nicely. +""" + +import copy +import dataclasses +import glob +import hashlib +import importlib +import inspect +import json +import os +import pathlib +import re +import secrets +from typing import ( + List, + Tuple, + Any, + Optional, + Type, + cast, + Dict, + NamedTuple, + Iterator, + Set, +) + +import docutils.nodes +import docutils.parsers.rst.states +import docutils.statemachine +import sphinx +import sphinx.addnodes +import sphinx.application +import sphinx.domains.python +import sphinx.environment +import sphinx.ext.autodoc +import sphinx.ext.autodoc.directive +import sphinx.ext.napoleon.docstring +import sphinx.pycode +import sphinx.util.docstrings +import sphinx.util.docutils +import sphinx.util.inspect +import sphinx.util.logging +import sphinx.util.typing + +from . import apidoc_formatting +from . import sphinx_utils + +logger = sphinx.util.logging.getLogger(__name__) + +_UNCONDITIONALLY_DOCUMENTED_MEMBERS = frozenset( + [ + "__init__", + "__class_getitem__", + "__call__", + "__getitem__", + "__setitem__", + ] +) +"""Special members to include even if they have no docstring.""" + + +class ParsedOverload(NamedTuple): + """Parsed representation of a single overload. + + For non-function types and non-overloaded functions, this just represents the + object itself. + + Sphinx does not really support pybind11-style overloaded functions directly. + It has minimal support functions with multiple signatures, with a single + docstring. However, pybind11 produces overloaded functions each with their + own docstring. This module adds support for documenting each overload as an + independent function. + + Additionally, we need a way to identify each overload, for the purpose of + generating a page name, listing in the table of contents sidebar, and + cross-referencing. Sphinx does not have a native solution to this problem + because it is not designed to support overloads. Doxygen uses some sort of + hash as the identifier, but that means links break with even minor changes to + the signature. + + Instead, we require that a unique id be manually assigned to each overload, + and specified as: + + Overload: + XXX + + in the docstring. Then the overload will be identified as + `module.Class.function(overload)`, and will be documented using the page name + `module.Class.function-overload`. Typically the overload id should be chosen + to be a parameter name that is unique to the overload. + """ + + doc: Optional[str] + """Docstring for individual overload. First line is the signature.""" + + overload_id: Optional[str] = None + """Overload id specified in the docstring. + + If there is just a single overload, will be `None`. Otherwise, if no overload + id is specified, a warning is produced and the index of the overload, + i.e. "1", "2", etc., is used as the id. + """ + + +def _extract_field(doc: str, field: str) -> Tuple[str, Optional[str]]: + pattern = f"\n\\s*\n{field}:\\s*\n\\s+([^\n]+)\n" + m = re.search(pattern, doc) + if m is None: + return doc, None + start, end = m.span() + return f"{doc[:start]}\n\n{doc[end:]}", m.group(1).strip() + + +_OVERLOADED_FUNCTION_RE = "^([^(]+)\\([^\n]*\nOverloaded function.\n" + + +def _parse_overloaded_function_docstring(doc: Optional[str]) -> List[ParsedOverload]: + """Parses a pybind11 overloaded function docstring. + + If the docstring is not for an overloaded function, just returns the full + docstring as a single "overload". + + :param doc: Original docstring. + :returns: List of parsed overloads. + :raises ValueError: If docstring has unexpected format. + """ + + if doc is None: + return [ParsedOverload(doc=doc, overload_id=None)] + m = re.match(_OVERLOADED_FUNCTION_RE, doc) + if m is None: + # Non-overloaded function + doc, overload_id = _extract_field(doc, "Overload") + return [ParsedOverload(doc=doc, overload_id=overload_id)] + + display_name = m.group(1) + doc = doc[m.end() :] + i = 1 + + def get_prefix(i: int): + return "\n%d. %s(" % (i, display_name) + + prefix = get_prefix(i) + parts: List[ParsedOverload] = [] + while doc: + if not doc.startswith(prefix): + raise ValueError( + "Docstring does not contain %r as expected: %r" + % ( + prefix, + doc, + ) + ) + doc = doc[len(prefix) - 1 :] + nl_index = doc.index("\n") + part_sig = doc[:nl_index] + doc = doc[nl_index + 1 :] + i += 1 + prefix = get_prefix(i) + end_index = doc.find(prefix) + if end_index == -1: + part = doc + doc = "" + else: + part = doc[:end_index] + doc = doc[end_index:] + + part, overload_id = _extract_field(part, "Overload") + if overload_id is None: + overload_id = str(i - 1) + + part_doc_with_sig = f"{display_name}{part_sig}\n{part}" + parts.append( + ParsedOverload( + doc=part_doc_with_sig, + overload_id=overload_id, + ) + ) + return parts + + +def _get_overloads_from_documenter( + documenter: sphinx.ext.autodoc.Documenter, +) -> List[ParsedOverload]: + docstring = sphinx.util.inspect.getdoc( + documenter.object, + documenter.get_attr, + documenter.env.config.autodoc_inherit_docstrings, + documenter.parent, + documenter.object_name, + ) + return _parse_overloaded_function_docstring(docstring) + + +def _has_default_value(node: sphinx.addnodes.desc_parameter): + for sub_node in node.traverse(condition=docutils.nodes.literal): + if "default_value" in sub_node.get("classes"): + return True + return False + + +def _summarize_signature( + env: sphinx.environment.BuildEnvironment, node: sphinx.addnodes.desc_signature +) -> None: + """Shortens a signature line to fit within `wrap_signatures_column_limit.""" + + obj_desc = node.parent + + options = apidoc_formatting.get_object_description_options( + env, obj_desc["domain"], obj_desc["objtype"] + ) + column_limit = options["wrap_signatures_column_limit"] + + def _must_shorten(): + return len(node.astext()) > column_limit + + parameterlist: Optional[sphinx.addnodes.desc_parameterlist] = None + for parameterlist in node.traverse(condition=sphinx.addnodes.desc_parameterlist): + break + + if parameterlist is None: + # Can't shorten a signature without a parameterlist + return + + # Remove initial `self` parameter + if parameterlist.children and parameterlist.children[0].astext() == "self": + del parameterlist.children[0] + + added_ellipsis = False + for next_parameter_index in range(len(parameterlist.children) - 1, -1, -1): + if not _must_shorten(): + return + + # First remove type annotation of last parameter, but only if it doesn't + # have a default value. + last_parameter = parameterlist.children[next_parameter_index] + if isinstance( + last_parameter, sphinx.addnodes.desc_parameter + ) and not _has_default_value(last_parameter): + del last_parameter.children[1:] + if not _must_shorten(): + return + + # Elide last parameter entirely + del parameterlist.children[next_parameter_index] + if not added_ellipsis: + added_ellipsis = True + ellipsis_node = sphinx.addnodes.desc_sig_punctuation("", "...") + param = sphinx.addnodes.desc_parameter() + param += ellipsis_node + parameterlist += param + + +class _MemberDocumenterEntry(NamedTuple): + """Represents a member of some outer scope (module/class) to document.""" + + documenter: sphinx.ext.autodoc.Documenter + is_attr: bool + name: str + """Member name within parent, e.g. class member name.""" + + full_name: str + """Full name under which to document the member. + + For example, "modname.ClassName.method". + """ + + parent_canonical_full_name: str + + overload: Optional[ParsedOverload] = None + + is_inherited: bool = False + """Indicates whether this is an inherited member.""" + + subscript: bool = False + """Whether this is a "subscript" method to be shown with [] instead of ().""" + + @property + def overload_suffix(self): + if self.overload and self.overload.overload_id: + return f"({self.overload.overload_id})" + return "" + + @property + def toc_title(self): + return self.name + self.overload_suffix + + +class _ApiEntityMemberReference(NamedTuple): + name: str + canonical_object_name: str + parent_canonical_object_name: str + inherited: bool = False + + +@dataclasses.dataclass +class _ApiEntity: + canonical_full_name: str + objtype: str + directive: str + overload_id: str + + @property + def canonical_object_name(self): + return self.canonical_full_name + self.overload_suffix + + @property + def object_name(self): + return self.documented_full_name + self.overload_suffix + + @property + def overload_suffix(self) -> str: + overload_id = self.overload_id + return f"({overload_id})" if overload_id else "" + + signatures: List[str] + options: Dict[str, str] + content: List[str] + group_name: str + order: Optional[int] + subscript: bool + + members: List[_ApiEntityMemberReference] + """Members of this entity.""" + + parents: List[_ApiEntityMemberReference] + """Parents that reference this entity as a member. + + Inverse of `members`.""" + + docname: str = "" + + documented_full_name: str = "" + documented_name: str = "" + top_level: bool = False + + base_classes: Optional[List[str]] = None + """List of base classes, as rST cross references.""" + + +def _is_constructor_name(name: str) -> bool: + return name in ("__init__", "__new__", "__class_getitem__") + + +class _ApiData: + + entities: Dict[str, _ApiEntity] + top_level_groups: Dict[str, List[_ApiEntityMemberReference]] + + def __init__(self): + self.entities = {} + self.top_level_groups = {} + + def get_name_for_signature( + self, entity: _ApiEntity, member: Optional[_ApiEntityMemberReference] + ) -> str: + if member is not None: + assert member.canonical_object_name == entity.canonical_object_name + # Get name for summary + if _is_constructor_name(member.name): + parent_entity = self.entities.get(member.parent_canonical_object_name) + if parent_entity is not None and parent_entity.objtype == "class": + # Display as the parent class name. + return parent_entity.documented_name + # Display as the member name + return member.name + full_name = entity.documented_full_name + if _is_constructor_name(entity.documented_name): + full_name = full_name[: -len(entity.documented_name) - 1] + module = entity.options.get("module") + if module is not None and full_name.startswith(module + "."): + # Strip module name, since it is specified separately. + full_name = full_name[len(module) + 1 :] + return full_name + + def sort_members( + self, members: List[_ApiEntityMemberReference], alphabetical=False + ): + def get_key(member: _ApiEntityMemberReference): + member_entity = self.entities[member.canonical_object_name] + order = member_entity.order or 0 + if alphabetical: + return (order, member.name.lower(), member.name) + return order + + members.sort(key=get_key) + + +def _ensure_module_name_in_signature(signode: sphinx.addnodes.desc_signature) -> None: + """Ensures non-summary objects are documented with the module name. + + Sphinx by default excludes the module name from class members, and does not + provide an option to override that. Since we display all objects on separate + pages, we want to include the module name for clarity. + + :param signode: Signature to modify in place. + """ + for node in signode.traverse(condition=sphinx.addnodes.desc_addname): + modname = signode.get("module") + if modname and not node.astext().startswith(modname + "."): + node.insert(0, docutils.nodes.Text(modname + ".")) + break + + +def _get_group_name( + default_groups: List[Tuple[re.Pattern, str]], entity: _ApiEntity +) -> str: + """Returns a default group name for an entry. + + This is used if the group name is not explicitly specified via "Group:" in the + docstring. + + :param default_groups: Default group associations. + :param entity: Entity to document. + :returns: The group name. + """ + s = f"{entity.objtype}:{entity.documented_full_name}" + group = "Public members" + for pattern, default_group in default_groups: + if pattern.fullmatch(s) is not None: + group = default_group + return group + + +def _get_order(default_order: List[Tuple[re.Pattern, int]], entity: _ApiEntity) -> int: + """Returns a default order value for an entry. + + This is used if the order is not explicitly specified via "Order:" in the + docstring. + + :param default_order: Default order associations. + :param entity: Entity to document. + :returns: The order. + """ + s = f"{entity.objtype}:{entity.documented_full_name}" + order = 0 + for pattern, order_value in default_order: + if pattern.fullmatch(s) is not None: + order = order_value + return order + + +def _mark_subscript_parameterlist(signode: sphinx.addnodes.desc_signature) -> None: + """Modifies an object description to display as a "subscript method". + + A "subscript method" is a property that defines __getitem__ and is intended to + be treated as a method invoked using [] rather than (), in order to allow + subscript syntax like ':'. + + :param node: Signature to modify in place. + """ + for sub_node in signode.traverse(condition=sphinx.addnodes.desc_parameterlist): + sub_node["parens"] = ("[", "]") + + +def _clean_init_signature(signode: sphinx.addnodes.desc_signature) -> None: + """Modifies an object description of an __init__ method. + + Removes the return type (always None) and the self parameter (since these + methods are displayed as the class name, without showing __init__). + + :param node: Signature to modify in place. + """ + # Remove first parameter. + for param in signode.traverse(condition=sphinx.addnodes.desc_parameter): + if param.children[0].astext() == "self": + param.parent.remove(param) + break + + # Remove return type. + for node in signode.traverse(condition=sphinx.addnodes.desc_returns): + node.parent.remove(node) + + +def _clean_class_getitem_signature(signode: sphinx.addnodes.desc_signature) -> None: + """Modifies an object description of a __class_getitem__ method. + + Removes the `static` prefix since these methods are shown using the class + name (i.e. as "subscript" constructors). + + :param node: Signature to modify in place. + + """ + # Remove `static` prefix + for prefix in signode.traverse(condition=sphinx.addnodes.desc_annotation): + prefix.parent.remove(prefix) + break + + +def _get_api_data( + env: sphinx.environment.BuildEnvironment, +) -> _ApiData: + return getattr(env, "_sphinx_immaterial_python_apigen_data") + + +def _generate_entity_desc_node( + env: sphinx.environment.BuildEnvironment, + entity: _ApiEntity, + state: docutils.parsers.rst.states.RSTState, + member: Optional[_ApiEntityMemberReference] = None, + callback=None, +): + api_data = _get_api_data(env) + + summary = member is not None + name = api_data.get_name_for_signature(entity, member) + + def object_description_transform( + app: sphinx.application.Sphinx, + domain: str, + objtype: str, + contentnode: sphinx.addnodes.desc_content, + ) -> None: + env = app.env + assert env is not None + + obj_desc = contentnode.parent + assert isinstance(obj_desc, sphinx.addnodes.desc) + signodes = cast(List[sphinx.addnodes.desc_signature], obj_desc[:-1]) + if not signodes: + return + + for signode in signodes: + fullname = signode["fullname"] + modname = signode["module"] + object_name = (modname + "." if modname else "") + fullname + if object_name != entity.object_name: + # This callback may be invoked for additional members + # documented within the body of `entry`, but we don't want + # to transform them here. + return + assert isinstance(signode, sphinx.addnodes.desc_signature) + if entity.subscript: + _mark_subscript_parameterlist(signode) + + if entity.documented_name in ("__init__", "__new__"): + _clean_init_signature(signode) + if entity.documented_name == "__class_getitem__": + _clean_class_getitem_signature(signode) + + if summary: + obj_desc["classes"].append("summary") + assert app.env is not None + _summarize_signature(app.env, signode) + + base_classes = entity.base_classes + if base_classes: + signode += sphinx.addnodes.desc_sig_punctuation("", "(") + for i, base_class in enumerate(base_classes): + base_entity = api_data.entities.get(base_class) + if base_entity is not None: + base_class = base_entity.object_name + if i != 0: + signode += sphinx.addnodes.desc_sig_punctuation("", ",") + signode += sphinx.addnodes.desc_sig_space() + signode += sphinx.domains.python._parse_annotation(base_class, env) + signode += sphinx.addnodes.desc_sig_punctuation("", ")") + + if callback is not None: + callback(contentnode) + + listener_id = env.app.connect( + "object-description-transform", object_description_transform, priority=999 + ) + content = entity.content + options = dict(entity.options) + options["nonodeid"] = "" + options["object-ids"] = json.dumps([entity.object_name] * len(entity.signatures)) + if summary: + content = _summarize_rst_content(content) + options["noindex"] = "" + options.pop("canonical", None) + else: + options["canonical"] = entity.canonical_object_name + try: + nodes = [ + x + for x in sphinx_utils.parse_rst( + state=state, + text=sphinx_utils.format_directive( + entity.directive, + signatures=[name + sig for sig in entity.signatures], + content="\n".join(content), + options=options, + ), + source_path=entity.object_name, + ) + if isinstance(x, sphinx.addnodes.desc) + ] + finally: + env.app.disconnect(listener_id) + + if len(nodes) != 1: + raise ValueError("Failed to document entity: %r" % (entity.object_name,)) + node = nodes[0] + + return node + + +def _generate_entity_summary( + env: sphinx.environment.BuildEnvironment, + member: _ApiEntityMemberReference, + state: docutils.parsers.rst.states.RSTState, + include_in_toc: bool, +) -> sphinx.addnodes.desc: + api_data = _get_api_data(env) + entity = api_data.entities[member.canonical_object_name] + objdesc = _generate_entity_desc_node( + env=env, entity=entity, member=member, state=state + ) + for sig_node in cast(List[sphinx.addnodes.desc_signature], objdesc.children[:-1]): + # Insert a link around the `desc_name` field + for sub_node in sig_node.traverse(condition=sphinx.addnodes.desc_name): + if include_in_toc: + sub_node['classes'].append('pseudo-toc-entry') + xref_node = sphinx.addnodes.pending_xref( + "", + sub_node.deepcopy(), + refdomain="py", + reftype="obj", + reftarget=entity.object_name, + refwarn=True, + refexplicit=True, + ) + sub_node.replace_self(xref_node) + break + # Only mark first signature as a `pseudo-toc-entry`. + include_in_toc = False + return objdesc + + +def _generate_group_summary( + env: sphinx.environment.BuildEnvironment, + members: List[_ApiEntityMemberReference], + state: docutils.parsers.rst.states.RSTState, + notoc: Optional[bool] = None, +): + + data = _get_api_data(env) + + nodes: List[docutils.nodes.Node] = [] + toc_entries: List[Tuple[str, str]] = [] + + for member in members: + member_entity = data.entities[member.canonical_object_name] + include_in_toc = True + if notoc is True: + include_in_toc = False + elif member is not member_entity.parents[0]: + include_in_toc = False + node = _generate_entity_summary(env=env, member=member, state=state, include_in_toc=include_in_toc) + if node is None: + continue + nodes.append(node) + + if include_in_toc: + toc_entries.append( + (member.name + member_entity.overload_suffix, member_entity.docname) + ) + + nodes.extend( + sphinx_utils.make_toctree_node( + state, + toc_entries, + options={"hidden": True}, + source_path="sphinx-immaterial-python-apigen", + ) + ) + return nodes + + +def _add_group_summary( + env: sphinx.environment.BuildEnvironment, + contentnode: docutils.nodes.Element, + sections: Dict[str, docutils.nodes.section], + group_name: str, + members: List[_ApiEntityMemberReference], + state: docutils.parsers.rst.states.RSTState, +) -> None: + + group_id = docutils.nodes.make_id(group_name) + section = sections.get(group_id) + if section is None: + section = docutils.nodes.section() + section["ids"].append(group_id) + title = docutils.nodes.title("", group_name) + section += title + contentnode += section + sections[group_id] = section + + section.extend(_generate_group_summary(env=env, members=members, state=state)) + + +def _merge_summary_nodes_into( + env: sphinx.environment.BuildEnvironment, + entity: _ApiEntity, + contentnode: docutils.nodes.Element, + state: docutils.parsers.rst.states.RSTState, +) -> None: + """Merges the member summary into `contentnode`. + + Members are organized into groups. The group is either specified explicitly + by a `Group:` field in the docstring, or determined automatically by + `_get_group_name`. If there is an existing section, the member summary is + appended to it. Otherwise, a new section is created. + + :param contentnode: The existing container to which the member summaries will be + added. If `contentnode` contains sections, those sections correspond to + group names. + """ + + sections: Dict[str, docutils.nodes.section] = {} + for section in contentnode.traverse(condition=docutils.nodes.section): + if section["ids"]: + sections[section["ids"][0]] = section + + # Maps group name to the list of members. + groups: Dict[str, List[_ApiEntityMemberReference]] = {} + + entities = _get_api_data(env).entities + for member in entity.members: + member_entity = entities[member.canonical_object_name] + groups.setdefault(member_entity.group_name, []).append(member) + + for group_name, members in groups.items(): + _add_group_summary( + env=env, + contentnode=contentnode, + sections=sections, + group_name=group_name, + members=members, + state=state, + ) + + +class PythonApigenEntityPageDirective(sphinx.util.docutils.SphinxDirective): + """Documents an entity and summarizes its members, if applicable.""" + + has_content = False + final_argument_whitespace = True + required_arguments = 1 + optional_arguments = 0 + + def run(self) -> List[docutils.nodes.Node]: + entities = _get_api_data(self.env).entities + object_name = self.arguments[0] + entity = entities[object_name] + + objtype = entity.objtype + objdesc = _generate_entity_desc_node( + self.env, + entity, + state=self.state, + callback=lambda contentnode: _merge_summary_nodes_into( + env=self.env, + entity=entity, + contentnode=contentnode, + state=self.state, + ), + ) + if objdesc is None: + return [] + + for signode in objdesc.children[:-1]: + signode = cast(sphinx.addnodes.desc_signature, signode) + _ensure_module_name_in_signature(signode) + + # Wrap in a section + section = docutils.nodes.section() + section["ids"].append("") + # Sphinx treates the first child of a `section` node as the title, + # regardless of its type. We use a comment node to avoid adding a title + # that would be redundant with the object description. + section += docutils.nodes.comment("", entity.object_name) + section += objdesc + return [section] + + +class PythonApigenTopLevelGroupDirective(sphinx.util.docutils.SphinxDirective): + """Summarizes the members of a top-level group.""" + + has_content = False + final_argument_whitespace = True + required_arguments = 1 + optional_arguments = 0 + + option_spec = { + "notoc": docutils.parsers.rst.directives.flag, + } + + def run(self) -> List[docutils.nodes.Node]: + env = self.env + data = _get_api_data(env) + top_level_groups = data.top_level_groups + + group_id = docutils.nodes.make_id(self.arguments[0]) + members = data.top_level_groups.get(group_id) + if members is None: + logger.warning( + "No top-level Python API group named: %r, valid groups are: %r", + group_id, + list(data.top_level_groups.keys()), + location=self.state_machine.get_source_and_line(self.lineno), + ) + return [] + return _generate_group_summary( + env=env, members=members, state=self.state, notoc="notoc" in self.options + ) + + +class PythonApigenEntitySummaryDirective(sphinx.util.docutils.SphinxDirective): + """Generates a summary/link to a Python entity.""" + + has_content = False + final_argument_whitespace = True + required_arguments = 1 + optional_arguments = 0 + + option_spec = { + "notoc": docutils.parsers.rst.directives.flag, + } + + def run(self) -> List[docutils.nodes.Node]: + env = self.env + data = _get_api_data(env) + object_name = self.arguments[0] + entity = data.entities.get(object_name) + if entity is None: + logger.warning( + "No Python entity named: %r", + object_name, + location=self.state_machine.get_source_and_line(self.lineno), + ) + return [] + return _generate_group_summary( + env=env, + members=[entity.parents[0]], + state=self.state, + notoc="notoc" in self.options, + ) + + +class _FakeBridge(sphinx.ext.autodoc.directive.DocumenterBridge): + def __init__( + self, + env: sphinx.environment.BuildEnvironment, + tab_width: int, + extra_options: dict, + ) -> None: + settings = docutils.parsers.rst.states.Struct(tab_width=tab_width) + document = docutils.parsers.rst.states.Struct(settings=settings) + state = docutils.parsers.rst.states.Struct(document=document) + options = sphinx.ext.autodoc.Options() + options["undoc-members"] = True + options["class-doc-from"] = "class" + options.update(extra_options) + super().__init__( + env=env, + reporter=sphinx.util.docutils.NullReporter(), + options=options, + lineno=0, + state=state, + ) + + +_EXCLUDED_SPECIAL_MEMBERS = frozenset( + [ + "__module__", + "__abstractmethods__", + "__dict__", + "__weakref__", + "__class__", + "__base__", + # Exclude pickling members since they are never documented. + "__getstate__", + "__setstate__", + ] +) + + +def _create_documenter( + env: sphinx.environment.BuildEnvironment, + documenter_cls: Type[sphinx.ext.autodoc.Documenter], + name: str, + tab_width: int = 8, +) -> sphinx.ext.autodoc.Documenter: + """Creates a documenter for the given full object name. + + Since we are using the documenter independent of any autodoc directive, we use + a `_FakeBridge` as the documenter bridge, similar to the strategy used by + `sphinx.ext.autosummary`. + + :param env: Sphinx build environment. + :param documenter_cls: Documenter class to use. + :param name: Full object name, e.g. `my_module.MyClass.my_function`. + :param tab_width: Tab width setting to use when parsing docstrings. + :returns: The documenter object. + """ + extra_options = {} + if documenter_cls.objtype == "class": + extra_options["special-members"] = sphinx.ext.autodoc.ALL + bridge = _FakeBridge(env, tab_width=tab_width, extra_options=extra_options) + documenter = documenter_cls(bridge, name) + assert documenter.parse_name() + assert documenter.import_object() + try: + documenter.analyzer = sphinx.pycode.ModuleAnalyzer.for_module( + documenter.get_real_modname() + ) + # parse right now, to get PycodeErrors on parsing (results will + # be cached anyway) + documenter.analyzer.find_attr_docs() + except sphinx.pycode.PycodeError: + # no source file -- e.g. for builtin and C modules + documenter.analyzer = None # type: ignore[assignment] + return documenter + + +def _get_member_documenter( + parent: sphinx.ext.autodoc.Documenter, + member_name: str, + member_value: Any, + is_attr: bool, +) -> Optional[sphinx.ext.autodoc.Documenter]: + """Creates a documenter for the given member. + + :param parent: Parent documenter. + :param member_name: Name of the member. + :param member_value: Value of the member. + :param is_attr: Whether the member is an attribute. + :returns: The documenter object. + """ + classes = [ + cls + for cls in parent.documenters.values() + if cls.can_document_member(member_value, member_name, is_attr, parent) + ] + if not classes: + return None + # prefer the documenter with the highest priority + classes.sort(key=lambda cls: cls.priority) + full_mname = parent.modname + "::" + ".".join(parent.objpath + [member_name]) + documenter = _create_documenter( + env=parent.env, + documenter_cls=classes[-1], + name=full_mname, + tab_width=parent.directive.state.document.settings.tab_width, + ) + return documenter + + +def _include_member(member_name: str, member_value: Any, is_attr: bool) -> bool: + """Determines whether a member should be documented. + + :param member_name: Name of the member. + :param member_value: Value of the member. + :param is_attr: Whether the member is an attribute. + :returns: True if the member should be documented. + """ + del is_attr + if member_name == "__init__": + doc = getattr(member_value, "__doc__", None) + if isinstance(doc, str) and doc.startswith("Initialize self. "): + return False + elif member_name in ("__hash__", "__iter__"): + if member_value is None: + return False + return True + + +def _get_subscript_method( + parent_documenter: sphinx.ext.autodoc.Documenter, entry: _MemberDocumenterEntry +) -> Any: + """Checks for a property that defines a subscript method. + + A subscript method is a property like `Class.vindex` where `fget` has a return + type of `Class._Vindex`, which is a class type. + + :param parent_documenter: Parent documenter for `entry`. + :param entry: Entry to check. + :returns: The type object (e.g. `Class._Vindex`) representing the subscript + method, or None if `entry` does not define a subscript method. + """ + if not isinstance(entry.documenter, sphinx.ext.autodoc.PropertyDocumenter): + return None + retann = entry.documenter.retann + if not retann: + return None + + config = entry.documenter.config + pattern = config.python_apigen_subscript_method_types + match = pattern.fullmatch(retann) + if not match: + return None + + # Attempt to import value + try: + mem_documenter = _create_documenter( + env=entry.documenter.env, + documenter_cls=sphinx.ext.autodoc.ClassDocumenter, + name=retann, + ) + mem = mem_documenter.object + except ImportError: + return None + if not mem: + return None + getitem = getattr(mem, "__getitem__", None) + if getitem is None: + return None + + return mem + + +def _transform_member( + parent_documenter: sphinx.ext.autodoc.Documenter, entry: _MemberDocumenterEntry +) -> Iterator[_MemberDocumenterEntry]: + """Converts an individual member into a sequence of members to document. + + :param parent_documenter: The parent documenter. + :param entry: The original entry to document. For most entries we simply yield the + entry unmodified. For entries that correspond to subscript methods, + though, we yield the __getitem__ member (and __setitem__, if applicable) + separately. + :returns: Iterator over modified entries to document. + """ + if entry.name == "__class_getitem__": + entry = entry._replace(subscript=True) + + mem = _get_subscript_method(parent_documenter, entry) + if mem is None: + yield entry + return + retann = entry.documenter.retann + + for suffix in ("__getitem__", "__setitem__"): + method = getattr(mem, suffix, None) + if method is None: + continue + import_name = f"{retann}.{suffix}" + if import_name.startswith(entry.documenter.modname + "."): + import_name = ( + entry.documenter.modname + + "::" + + import_name[len(entry.documenter.modname) + 1 :] + ) + new_documenter = _create_documenter( + env=parent_documenter.env, + documenter_cls=sphinx.ext.autodoc.MethodDocumenter, + name=import_name, + tab_width=parent_documenter.directive.state.document.settings.tab_width, + ) + if suffix != "__getitem__": + new_member_name = f"{entry.name}.{suffix}" + full_name = f"{entry.full_name}.{suffix}" + subscript = False + else: + new_member_name = f"{entry.name}" + full_name = entry.full_name + subscript = True + + yield _MemberDocumenterEntry( + documenter=new_documenter, + name=new_member_name, + is_attr=False, + full_name=full_name, + parent_canonical_full_name=entry.parent_canonical_full_name, + subscript=subscript, + ) + + +def _prepare_documenter_docstring(entry: _MemberDocumenterEntry) -> None: + """Initializes `entry.documenter` with the correct docstring. + + This overrides the docstring based on `entry.overload` if applicable. + + This must be called before using `entry.documenter`. + + :param entry: Entry to prepare. + """ + + if entry.overload and ( + entry.overload.overload_id is not None + or + # For methods, we don't need `ModuleAnalyzer`, so it is safe to always + # override the normal mechanism of obtaining the docstring. + # Additionally, for `__init__` and `__new__` we need to specify the + # docstring explicitly to work around + # https://github.com/sphinx-doc/sphinx/pull/9518. + isinstance(entry.documenter, sphinx.ext.autodoc.MethodDocumenter) + ): + # Force autodoc to use the overload-specific signature. autodoc already + # has an internal mechanism for overriding the docstrings based on the + # `_new_docstrings` member. + tab_width = entry.documenter.directive.state.document.settings.tab_width + setattr( + entry.documenter, + "_new_docstrings", + [ + sphinx.util.docstrings.prepare_docstring( + entry.overload.doc or "", tabsize=tab_width + ) + ], + ) + else: + # Force autodoc to obtain the docstring through its normal mechanism, + # which includes the "ModuleAnalyzer" for reading docstrings of + # variables/attributes that are only contained in the source code. + setattr(entry.documenter, "_new_docstrings", None) + + # Workaround for https://github.com/sphinx-doc/sphinx/pull/9518 + orig_get_doc = entry.documenter.get_doc + + def get_doc(*args, **kwargs) -> List[List[str]]: + doc_strings = getattr(entry.documenter, "_new_docstrings", None) + if doc_strings is not None: + return doc_strings + return orig_get_doc(*args, **kwargs) # type: ignore + + entry.documenter.get_doc = get_doc # type: ignore[assignment] + + +def _is_conditionally_documented_entry(entry: _MemberDocumenterEntry): + if entry.name in _UNCONDITIONALLY_DOCUMENTED_MEMBERS: + return False + return sphinx.ext.autodoc.special_member_re.match(entry.name) + + +def _get_member_overloads( + entry: _MemberDocumenterEntry, +) -> Iterator[_MemberDocumenterEntry]: + """Returns the list of overloads for a given entry.""" + + if entry.name in _EXCLUDED_SPECIAL_MEMBERS: + return + + overloads = _get_overloads_from_documenter(entry.documenter) + for overload in overloads: + # Shallow copy the documenter. Certain methods on the documenter mutate it, + # and we don't want those mutations to affect other overloads. + documenter_copy = copy.copy(entry.documenter) + documenter_copy.options = documenter_copy.options.copy() + new_entry = entry._replace( + overload=overload, + documenter=documenter_copy, + ) + if _is_conditionally_documented_entry(new_entry): + # Only document this entry if it has a docstring. + _prepare_documenter_docstring(new_entry) + new_entry.documenter.format_signature() + doc = new_entry.documenter.get_doc() + if not doc: + continue + if not any(x for x in doc): + # No docstring, skip. + continue + + new_entry = entry._replace( + overload=overload, documenter=copy.copy(entry.documenter) + ) + + yield new_entry + + +def _get_documenter_direct_members( + documenter: sphinx.ext.autodoc.Documenter, + canonical_full_name: str, +) -> Iterator[_MemberDocumenterEntry]: + """Returns the sequence of direct members to document. + + The order is mostly determined by the definition order. + + This excludes inherited members. + + :param documenter: Documenter for which to obtain members. + :returns: Iterator over members to document. + """ + if not isinstance( + documenter, + (sphinx.ext.autodoc.ClassDocumenter, sphinx.ext.autodoc.ModuleDocumenter), + ): + # Only classes and modules have members. + return + + members_check_module, members = documenter.get_object_members(want_all=True) + del members_check_module + if members: + try: + # get_object_members does not preserve definition order, but __dict__ does + # in Python 3.6 and later. + member_dict = sphinx.util.inspect.safe_getattr( + documenter.object, "__dict__" + ) + member_order = {k: i for i, k in enumerate(member_dict.keys())} + members.sort(key=lambda entry: member_order.get(entry[0], float("inf"))) + except AttributeError: + pass + filtered_members = [ + x + for x in documenter.filter_members(members, want_all=True) + if _include_member(*x) + ] + for member_name, member_value, is_attr in filtered_members: + member_documenter = _get_member_documenter( + parent=documenter, + member_name=member_name, + member_value=member_value, + is_attr=is_attr, + ) + if member_documenter is None: + continue + full_name = f"{documenter.fullname}.{member_name}" + entry = _MemberDocumenterEntry( + cast(sphinx.ext.autodoc.Documenter, member_documenter), + is_attr, + parent_canonical_full_name=canonical_full_name, + name=member_name, + full_name=full_name, + ) + for transformed_entry in _transform_member(documenter, entry): + yield from _get_member_overloads(transformed_entry) + + +def _get_documenter_members( + documenter: sphinx.ext.autodoc.Documenter, + canonical_full_name: str, +) -> Iterator[_MemberDocumenterEntry]: + """Returns the sequence of members to document, including inherited members. + + :param documenter: Parent documenter for which to find members. + :returns: Iterator over members to document. + """ + seen_members: Set[str] = set() + + def _get_unseen_members( + members: Iterator[_MemberDocumenterEntry], is_inherited: bool + ) -> Iterator[_MemberDocumenterEntry]: + for member in members: + overload_name = member.toc_title + if overload_name in seen_members: + continue + seen_members.add(overload_name) + yield member._replace(is_inherited=is_inherited) + + yield from _get_unseen_members( + _get_documenter_direct_members( + documenter, canonical_full_name=canonical_full_name + ), + is_inherited=False, + ) + + if documenter.objtype != "class": + return + + for cls in inspect.getmro(documenter.object): + if cls is documenter.object: + continue + if cls.__module__ in ("builtins", "pybind11_builtins"): + continue + class_name = f"{cls.__module__}::{cls.__qualname__}" + parent_canonical_full_name = f"{cls.__module__}.{cls.__qualname__}" + try: + superclass_documenter = _create_documenter( + env=documenter.env, + documenter_cls=sphinx.ext.autodoc.ClassDocumenter, + name=class_name, + tab_width=documenter.directive.state.document.settings.tab_width, + ) + yield from _get_unseen_members( + _get_documenter_direct_members( + superclass_documenter, + canonical_full_name=parent_canonical_full_name, + ), + is_inherited=True, + ) + except Exception as e: # pylint: disable=broad-except + logger.warning( + "Cannot obtain documenter for base class %r of %r: %r", + cls, + documenter.fullname, + e, + ) + + +_ENTITY_PAGE_INITIAL_COMMENT = ( + "..\n DO NOT EDIT. GENERATED by sphinx_immaterial.apidoc.python.apigen.\n" +) + + +def _is_generated_file(rst_path: str) -> bool: + try: + if os.path.islink(rst_path) or not os.path.isfile(rst_path): + return False + content = pathlib.Path(rst_path).read_text(encoding="utf-8") + return content.startswith(_ENTITY_PAGE_INITIAL_COMMENT) + except: # pylint: disable=bare-except + return False + + +def _write_member_documentation_page( + app: sphinx.application.Sphinx, entity: _ApiEntity +) -> None: + """Writes the RST file that documents the given entity. + + This simply writes a `python-apigen-entity-page` directive to the generated + file. The actual documentation is generated by that directive. + + :param app: Sphinx application object. + :param entity: Entity to document. + """ + + rst_path = os.path.join(app.srcdir, entity.docname + ".rst") + + content = _ENTITY_PAGE_INITIAL_COMMENT + # Suppress "Edit this page" link since the page is generated. + content += "\n\n:hide-edit-link:\n\n" + content += sphinx_utils.format_directive( + "python-apigen-entity-page", + entity.object_name, + ) + if os.path.exists(rst_path): + logger.error( + "Generated documentation page for %r would overwrite existing source file %r", + entity.object_name, + rst_path, + ) + return + pathlib.Path(rst_path).write_text(content, encoding="utf-8") + + +class SplitAutodocRstOutput(NamedTuple): + directive: str + options: Dict[str, str] + content: List[str] + group_name: Optional[str] + order: Optional[int] + + +def _split_autodoc_rst_output( + rst_strings: docutils.statemachine.StringList, +) -> SplitAutodocRstOutput: + m = re.fullmatch(r"\.\. ([^:]+:[^:]+):: (.*)", rst_strings[1], re.DOTALL) + assert m is not None, repr(rst_strings[1]) + directive = m.group(1) + signatures = [m.group(2)] + signature_prefix = " " * (6 + len(directive)) + i = 2 + while i < len(rst_strings): + line = rst_strings[i] + if line.startswith(signature_prefix): + signatures.append(line[len(signature_prefix) :]) + i += 1 + else: + break + options: Dict[str, str] = {} + while i < len(rst_strings): + line = rst_strings[i] + m = re.fullmatch(r" :([^:]+):(.*)", line, re.DOTALL) + if m is None: + break + options[m.group(1)] = m.group(2).strip() + i += 1 + assert i < len(rst_strings) + assert rst_strings[i] == "" + + i += 1 + + while i < len(rst_strings) and not rst_strings[i].strip(): + i += 1 + + content_start_line = i + + def extract_field(name: str) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: + field_prefix = f" :{name}:" + for i in range(content_start_line, len(rst_strings)): + line = rst_strings[i] + if not line.startswith(field_prefix): + continue + value = line[len(field_prefix) :].strip() + location = rst_strings.items[i] + del rst_strings[i] + return value, location + return None, None + + group_name, group_location = extract_field("group") + + order_str, order_location = extract_field("order") + order = None + if order_str is not None: + try: + order = int(order_str) + except ValueError: + logger.error("Invalid order value: %r", order_str, location=order_location) + + # Strip 3 spaces of indent. + content = [line[3:] for line in rst_strings.data[content_start_line:]] + + return SplitAutodocRstOutput( + directive=directive, + options=options, + content=content, + group_name=group_name, + order=order, + ) + + +def _summarize_rst_content(content: List[str]) -> List[str]: + i = 0 + # Skip over blank lines before start of directive content + while i < len(content) and not content[i].strip(): + i += 1 + # Skip over first paragraph + while i < len(content) and content[i].strip(): + i += 1 + + return content[:i] + + +class _ApiEntityCollector: + def __init__( + self, + entities: Dict[str, _ApiEntity], + ): + self.entities = entities + + def collect_entity_recursively( + self, + entry: _MemberDocumenterEntry, + ) -> str: + canonical_full_name = None + if isinstance(entry.documenter, sphinx.ext.autodoc.ClassDocumenter): + canonical_full_name = entry.documenter.get_canonical_fullname() + if canonical_full_name is None: + canonical_full_name = f"{entry.parent_canonical_full_name}.{entry.name}" + + canonical_object_name = canonical_full_name + entry.overload_suffix + existing_entity = self.entities.get(canonical_object_name) + if existing_entity is not None: + return canonical_object_name + + if ( + entry.overload + and entry.overload.overload_id + and re.fullmatch("[0-9]+", entry.overload.overload_id) + ): + logger.warning("Unspecified overload id: %s", canonical_object_name) + + rst_strings = docutils.statemachine.StringList() + entry.documenter.directive.result = rst_strings + _prepare_documenter_docstring(entry) + # Prevent autodoc from also documenting members, since this extension does + # that separately. + def document_members(*args, **kwargs): + return + + entry.documenter.document_members = document_members # type: ignore[assignment] + entry.documenter.generate() + + base_classes: Optional[List[str]] = None + + if isinstance(entry.documenter, sphinx.ext.autodoc.ClassDocumenter): + # By default (unless the `autodoc_class_signature` config option is + # set to `"separated"`), autodoc will include the `__init__` + # parameters in the signature. Since that convention does not work + # well with this extension, we just bypass that here. + signatures = [""] + + if entry.documenter.config.python_apigen_show_base_classes: + obj = entry.documenter.object + bases = sphinx.util.inspect.getorigbases(obj) + if bases is None: + bases = getattr(obj, "__bases__", None) + if bases: + base_list = list(bases) + entry.documenter.env.events.emit( + "autodoc-process-bases", + entry.documenter.fullname, + obj, + entry.documenter.options, + base_list, + ) + base_classes = [ + sphinx.util.typing.stringify(base) + for base in base_list + if base is not object + ] + else: + signatures = entry.documenter.format_signature().split("\n") + + split_result = _split_autodoc_rst_output(rst_strings) + + overload_id: Optional[str] = None + if entry.overload is not None: + overload_id = entry.overload.overload_id + + entity = _ApiEntity( + documented_full_name="", + canonical_full_name=canonical_full_name, + objtype=entry.documenter.objtype, + group_name=split_result.group_name or "", + order=split_result.order, + directive=split_result.directive, + signatures=signatures, + options=split_result.options, + content=split_result.content, + members=[], + parents=[], + subscript=entry.subscript, + overload_id=overload_id or "", + base_classes=base_classes, + ) + + self.entities[canonical_object_name] = entity + + entity.members = self.collect_documenter_members( + entry.documenter, + canonical_object_name=canonical_object_name, + ) + + return canonical_object_name + + def collect_documenter_members( + self, + documenter: sphinx.ext.autodoc.Documenter, + canonical_object_name: str, + ) -> List[_ApiEntityMemberReference]: + members: List[_ApiEntityMemberReference] = [] + + for entry in _get_documenter_members( + documenter, canonical_full_name=canonical_object_name + ): + member_canonical_object_name = self.collect_entity_recursively(entry) + member = _ApiEntityMemberReference( + name=entry.name, + parent_canonical_object_name=canonical_object_name, + canonical_object_name=member_canonical_object_name, + inherited=entry.is_inherited, + ) + members.append(member) + child = self.entities[member_canonical_object_name] + child.parents.append(member) + + return members + + +def _get_base_docname(output_prefixes: Dict[str, str], full_name: str) -> str: + end_idx = len(full_name) + while True: + output_path = output_prefixes.get(full_name[:end_idx]) + if output_path is not None: + return output_path + full_name[end_idx + 1 :] + new_end_idx = full_name.rfind(".", 0, end_idx) + if new_end_idx == -1: + raise ValueError( + f"Could not find output prefix for {full_name!r} in {output_prefixes!r}" + ) + end_idx = new_end_idx + + +def _get_docname( + output_prefixes: Dict[str, str], + documented_full_name: str, + overload_id: str, + case_insensitive_filesystem: bool, +): + + docname = _get_base_docname(output_prefixes, documented_full_name) + + if case_insensitive_filesystem: + name_hash = hashlib.sha256( + os.path.basename(docname).encode("utf-8") + ).hexdigest()[:8] + docname += f"-{name_hash}" + + if overload_id: + docname += f"-{overload_id}" + + return docname + + +def _assign_documented_full_names( + entities: Dict[str, _ApiEntity], + default_groups: List[Tuple[re.Pattern, str]], + default_order: List[Tuple[re.Pattern, int]], + output_prefixes: Dict[str, str], + case_insensitive_filesystem: bool, +) -> None: + """Determines the full name under which each entity will be documented.""" + + def get_documented_full_name(entity: _ApiEntity) -> str: + documented_full_name = entity.documented_full_name + if documented_full_name: + # Name already assigned + return documented_full_name + + parents = entity.parents + assert len(parents) > 0 + + def parent_sort_key(parent_ref: _ApiEntityMemberReference): + canonical_name_from_parent = ( + parent_ref.parent_canonical_object_name + "." + parent_ref.name + ) + canonical = canonical_name_from_parent == entity.canonical_full_name + inherited = parent_ref.inherited + return (canonical is False, inherited, canonical_name_from_parent) + + parents.sort(key=parent_sort_key) + + parent_ref = parents[0] + parent_entity = entities.get(parent_ref.parent_canonical_object_name) + if parent_entity is None: + # Parent is a module. + parent_documented_name = parent_ref.parent_canonical_object_name + entity.top_level = True + else: + parent_documented_name = get_documented_full_name(parent_entity) + documented_full_name = parent_documented_name + "." + parent_ref.name + entity.documented_full_name = documented_full_name + entity.documented_name = parent_ref.name + + # Assign default group name. + if not entity.group_name: + entity.group_name = _get_group_name(default_groups, entity) + + if entity.order is None: + entity.order = _get_order(default_order, entity) + + entity.docname = _get_docname( + output_prefixes, + documented_full_name, + entity.overload_id, + case_insensitive_filesystem, + ) + + return documented_full_name + + for entity in list(entities.values()): + get_documented_full_name(entity) + entities[entity.object_name] = entity + + +def _is_case_insensitive_filesystem(path: str) -> bool: + suffix = secrets.token_hex(16) + temp_path = path + suffix + "a.rst" + try: + pathlib.Path(temp_path).write_text( + _ENTITY_PAGE_INITIAL_COMMENT, encoding="utf-8" + ) + return os.path.exists(path + suffix + "A.rst") + finally: + os.remove(temp_path) + + +def _builder_inited(app: sphinx.application.Sphinx) -> None: + """Generates the rST files for API members.""" + + env = app.env + assert env is not None + + data = _ApiData() + + setattr(env, "_sphinx_immaterial_python_apigen_data", data) + + apigen_modules = app.config.python_apigen_modules + if not apigen_modules: + return + + for module_name, output_path in apigen_modules.items(): + try: + importlib.import_module(module_name) + except ImportError: + logger.warning( + "Failed to import module %s specified in python_apigen_modules", + module_name, + exc_info=True, + ) + continue + + documenter = _create_documenter( + env=env, + documenter_cls=sphinx.ext.autodoc.ModuleDocumenter, + name=module_name, + ) + _ApiEntityCollector(entities=data.entities,).collect_documenter_members( + documenter=documenter, + canonical_object_name=module_name, + ) + + # Prepare output directories and determine if the filesystem is case + # insensitive. + seen_output_dirs = set() + case_insensitive_filesystem = app.config.python_apigen_case_insensitive_filesystem + for output_path in apigen_modules.values(): + output_dir = os.path.dirname(os.path.join(app.srcdir, output_path)) + if output_dir in seen_output_dirs: + continue + seen_output_dirs.add(output_dir) + os.makedirs(output_dir, exist_ok=True) + if case_insensitive_filesystem is None: + if _is_case_insensitive_filesystem(os.path.join(app.srcdir, output_path)): + case_insensitive_filesystem = True + if case_insensitive_filesystem is None: + case_insensitive_filesystem = False + + default_groups = [ + (re.compile(pattern), group) + for pattern, group in app.config.python_apigen_default_groups + ] + + default_order = [ + (re.compile(pattern), order) + for pattern, order in app.config.python_apigen_default_order + ] + + _assign_documented_full_names( + entities=data.entities, + default_groups=default_groups, + default_order=default_order, + output_prefixes=apigen_modules, + case_insensitive_filesystem=case_insensitive_filesystem, + ) + + # Remove stale generated files. + srcdir = app.srcdir + for _, output_path in app.config.python_apigen_modules.items(): + glob_pattern = os.path.join(srcdir, output_path) + for p in glob.glob(os.path.join(srcdir, output_path + "*.rst"), recursive=True): + if not _is_generated_file(p): + continue + try: + os.remove(p) + except OSError as e: + logger.warning("Failed to remove stale generated file %r: %s", p, e) + + # Write page for each entity. + all_pages: Dict[str, str] = {} + + alphabetical = app.config.python_apigen_order_tiebreaker == "alphabetical" + + for object_name, entity in data.entities.items(): + if object_name != entity.object_name: + # Alias + continue + + data.sort_members(entity.members, alphabetical=alphabetical) + + rst_path = entity.docname + ".rst" + if rst_path in all_pages: + logger.error( + "Both %r and %r map to generated path %r", + all_pages[rst_path], + entity.object_name, + rst_path, + ) + continue + _write_member_documentation_page(app, entity) + all_pages[rst_path] = entity.object_name + if entity.top_level: + group_id = docutils.nodes.make_id(entity.group_name) + data.top_level_groups.setdefault(group_id, []).append(entity.parents[0]) + + for members in data.top_level_groups.values(): + data.sort_members(members, alphabetical=alphabetical) + + +def _monkey_patch_napoleon_to_add_group_field(): + """Adds support to sphinx.ext.napoleon for the "Group" and "Order" fields. + + This field is used by this module to organize members into groups. + """ + orig_load_custom_sections = ( + sphinx.ext.napoleon.docstring.GoogleDocstring._load_custom_sections + ) # pylint: disable=protected-access + + def parse_section( + self: sphinx.ext.napoleon.docstring.GoogleDocstring, section: str + ) -> List[str]: + lines = self._strip_empty( + self._consume_to_next_section() + ) # pylint: disable=protected-access + lines = self._dedent(lines) # pylint: disable=protected-access + name = section.lower() + if len(lines) != 1: + raise ValueError(f"Expected exactly one {name} in {section} section") + return [f":{name}: " + lines[0], ""] + + def load_custom_sections( + self: sphinx.ext.napoleon.docstring.GoogleDocstring, + ) -> None: + orig_load_custom_sections(self) + self._sections["group"] = lambda section: parse_section( + self, section + ) # pylint: disable=protected-access + self._sections["order"] = lambda section: parse_section( + self, section + ) # pylint: disable=protected-access + + sphinx.ext.napoleon.docstring.GoogleDocstring._load_custom_sections = ( + load_custom_sections # pylint: disable=protected-access + ) + + +def _config_inited( + app: sphinx.application.Sphinx, config: sphinx.config.Config +) -> None: + if isinstance(config.python_apigen_subscript_method_types, str): + setattr( + config, + "python_apigen_subscript_method_types", + re.compile(config.python_apigen_subscript_method_types), + ) + + +def setup(app: sphinx.application.Sphinx): + """Initializes the extension.""" + _monkey_patch_napoleon_to_add_group_field() + app.connect("builder-inited", _builder_inited) + app.connect("config-inited", _config_inited) + app.setup_extension("sphinx.ext.autodoc") + app.add_directive("python-apigen-entity-page", PythonApigenEntityPageDirective) + app.add_directive("python-apigen-group", PythonApigenTopLevelGroupDirective) + app.add_directive( + "python-apigen-entity-summary", PythonApigenEntitySummaryDirective + ) + app.add_config_value( + "python_apigen_modules", types=(Dict[str, str],), default={}, rebuild="env" + ) + app.add_config_value( + "python_apigen_default_groups", + types=(List[Tuple[str, str]],), + default=[(".*", "Public members"), ("class:.*", "Classes")], + rebuild="env", + ) + app.add_config_value( + "python_apigen_default_order", + types=(List[Tuple[str, int]],), + default=[], + rebuild="env", + ) + app.add_config_value( + "python_apigen_order_tiebreaker", + types=sphinx.config.ENUM("definition_order", "alphabetical"), + default="definition_order", + rebuild="env", + ) + app.add_config_value( + "python_apigen_subscript_method_types", + default=r".*\._[^.]*", + types=(re.Pattern,), + rebuild="env", + ) + app.add_config_value( + "python_apigen_case_insensitive_filesystem", + default=None, + types=(bool, type(None)), + rebuild="env", + ) + app.add_config_value( + "python_apigen_show_base_classes", + default=True, + types=(bool,), + rebuild="env", + ) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/sphinx_immaterial/sphinx_utils.py b/sphinx_immaterial/sphinx_utils.py index b8d35c6bb..0cac52bfd 100644 --- a/sphinx_immaterial/sphinx_utils.py +++ b/sphinx_immaterial/sphinx_utils.py @@ -1,11 +1,12 @@ """Utilities for use with Sphinx.""" import io -from typing import Optional, Dict, Union, List +from typing import Optional, Dict, Union, List, Tuple, Mapping, Sequence import docutils.nodes import docutils.parsers.rst.states import docutils.statemachine +import sphinx.addnodes import sphinx.util.docutils from typing_extensions import Literal @@ -31,13 +32,15 @@ def to_statemachine_stringlist( def format_directive( name: str, *args: str, + signatures: Optional[Sequence[str]] = None, content: Optional[str] = None, - options: Optional[Dict[str, Union[None, str, bool]]] = None, + options: Optional[Mapping[str, Union[None, str, bool]]] = None, ) -> str: """Formats a RST directive into RST syntax. :param name: Directive name, e.g. "json:schema". :param args: List of directive arguments. + :param signatures: List of signatures, alternative to ``args``. :param content: Directive body content. :param options: Directive options. @@ -45,9 +48,15 @@ def format_directive( """ out = io.StringIO() out.write("\n\n") - out.write(f".. {name}::") - for arg in args: - out.write(f" {arg}") + prefix = f".. {name}:: " + out.write(prefix) + assert not args or not signatures + out.write(" ".join(args)) + if signatures: + for i, signature in enumerate(signatures): + if i > 0: + out.write("\n" + " " * len(prefix)) + out.write(signature) out.write("\n") if options: for key, value in options.items(): @@ -156,3 +165,30 @@ def summarize_element_text( text = text[: sentence_end + 1] text = text.replace("\n", " ") return text.strip() + + +def make_toctree_node( + state: docutils.parsers.rst.states.RSTState, + toc_entries: List[Tuple[str, str]], + options: dict, + source_path: str, + source_line: int = 0, +) -> List[docutils.nodes.Node]: + # The Sphinx `toctree` directive parser cannot handle page names that + # include angle brackets. Therefore, we use the directive to create an + # empty toctree node and then add the entries directly. + toctree_nodes = parse_rst( + state=state, + text=format_directive("toctree", options=options), + source_path=source_path, + source_line=source_line, + ) + toctree: Optional[sphinx.addnodes.toctree] = None + for node in toctree_nodes[-1].traverse(condition=sphinx.addnodes.toctree): + toctree = node + break + if toctree is None: + raise ValueError("No toctree node found") + toctree["entries"].extend(toc_entries) + toctree["includefiles"].extend([path for _, path in toc_entries]) + return toctree_nodes diff --git a/src/assets/stylesheets/main/layout/_nav.scss b/src/assets/stylesheets/main/layout/_nav.scss index 13f337c28..b31594fdf 100644 --- a/src/assets/stylesheets/main/layout/_nav.scss +++ b/src/assets/stylesheets/main/layout/_nav.scss @@ -40,7 +40,11 @@ // Navigation title &__title { - display: block; + // sphinx-immaterial: display object description icon as a + // separate column, don't allow title text to wrap underneath it. + display: flex; + // sphinx-immaterial: ensure icons are centered vertically with text. + align-items: center; padding: 0 px2rem(12px); overflow: hidden; font-weight: 700; @@ -389,6 +393,12 @@ // are truncated with an ellipsis rather than wrapping. .md-nav__title .md-ellipsis { white-space: nowrap; + + // In Chrome, elements override `white-space: nowrap`, but + // setting them to `display: none` prevents that. + wbr { + display: none; + } } } diff --git a/tests/python_apigen_test.py b/tests/python_apigen_test.py new file mode 100644 index 000000000..5cc976bad --- /dev/null +++ b/tests/python_apigen_test.py @@ -0,0 +1,82 @@ +import pathlib + +import pytest + +from sphinx_immaterial.python_apigen import _get_api_data + +from sphinx.testing.path import path as SphinxPath + +pytest_plugins = ("sphinx.testing.fixtures",) + + +@pytest.fixture +def apigen_make_app(tmp_path: pathlib.Path, make_app): + + conf = """ +extensions = [ + "sphinx_immaterial", + "sphinx_immaterial.python_apigen", +] +html_theme = "sphinx_immaterial" +""" + + def make(extra_conf: str = "", **kwargs): + (tmp_path / "conf.py").write_text(conf + extra_conf, encoding="utf-8") + (tmp_path / "index.rst").write_text("", encoding="utf-8") + return make_app(srcdir=SphinxPath(str(tmp_path)), **kwargs) + + yield make + + +@pytest.mark.parametrize( + "order_tiebreaker,expected_members", + [("alphabetical", ["a", "b"]), ("definition_order", ["b", "a"])], +) +def test_alphabetical(apigen_make_app, order_tiebreaker, expected_members): + + app = apigen_make_app( + confoverrides=dict( + python_apigen_order_tiebreaker=order_tiebreaker, + python_apigen_default_groups=[], + python_apigen_modules={ + "python_apigen_test_modules.alphabetical": "api/", + }, + ), + ) + + print(app._status.getvalue()) + print(app._warning.getvalue()) + + data = _get_api_data(app.env) + members = [x.name for x in data.top_level_groups["public-members"]] + assert members == expected_members + + +def test_classmethod(apigen_make_app): + testmod = "python_apigen_test_modules.classmethod" + app = apigen_make_app( + confoverrides=dict( + python_apigen_default_groups=[ + ("method:.*", "Methods"), + ("classmethod:.*", "Class methods"), + ("staticmethod:.*", "Static methods"), + ], + python_apigen_modules={ + testmod: "api/", + }, + ), + ) + + print(app._status.getvalue()) + print(app._warning.getvalue()) + + data = _get_api_data(app.env) + + # FIXME: Currently all methods are assigned to the 'Methods" group because + # the `classmethod` and `staticmethod` object types aren't actually used by + # Sphinx. + # + # https://github.com/sphinx-doc/sphinx/issues/3743 + assert data.entities[f"{testmod}.Foo.my_method"].group_name == "Methods" + assert data.entities[f"{testmod}.Foo.my_staticmethod"].group_name == "Methods" + assert data.entities[f"{testmod}.Foo.my_classmethod"].group_name == "Methods" diff --git a/tests/python_apigen_test_modules/__init__.py b/tests/python_apigen_test_modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/python_apigen_test_modules/alphabetical.py b/tests/python_apigen_test_modules/alphabetical.py new file mode 100644 index 000000000..f43a1d883 --- /dev/null +++ b/tests/python_apigen_test_modules/alphabetical.py @@ -0,0 +1,6 @@ +def b(): + pass + + +def a(): + pass diff --git a/tests/python_apigen_test_modules/classmethod.py b/tests/python_apigen_test_modules/classmethod.py new file mode 100644 index 000000000..59058480a --- /dev/null +++ b/tests/python_apigen_test_modules/classmethod.py @@ -0,0 +1,11 @@ +class Foo: + @classmethod + def my_classmethod(cls, arg: str) -> int: + pass + + @staticmethod + def my_staticmethod(arg: str) -> int: + pass + + def my_method(self, arg: str) -> int: + pass diff --git a/tools/build/index.ts b/tools/build/index.ts index fb78b042d..600e9ba3d 100644 --- a/tools/build/index.ts +++ b/tools/build/index.ts @@ -216,7 +216,7 @@ const docs$ = (() => { let dirty = false return defer(() => process.argv.includes("--watch") ? watch(["docs/**", "sphinx_immaterial/**"], - { ignored: ["*.pyc", "docs/_build/**", "docs/generated/**"] }) + { ignored: ["*.pyc", "docs/_build/**", "docs/python_apigen_generated/**"] }) : EMPTY ).pipe(startWith("*"), switchMap(async () => { @@ -237,7 +237,16 @@ const docs$ = (() => { } }) }) - const child = spawn("sphinx-build", ["docs", "docs/_build", "-a"], + await new Promise((resolve, reject) => { + rimraf("docs/python_apigen_generated", error => { + if (error != null) { + reject(error) + } else { + resolve(undefined) + } + }) + }) + const child = spawn("sphinx-build", ["docs", "docs/_build", "-a", "-j", "auto"], {stdio: "inherit"}) await new Promise(resolve => { child.on("exit", resolve) From 3fc3dfd31ead40f7d2d554ecb676a1f6daeb904f Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Wed, 29 Jun 2022 12:40:59 -0700 Subject: [PATCH 16/26] Add toc.follow-like functionality This adds an independent implementation of the toc.follow feature that is available in mkdocs-material-insiders. Unlike the mkdocs-material feature, this also scrolls the left-side panel to keep the current document/current section within the viewport. --- docs/conf.py | 3 +- docs/customization.rst | 28 +++++++ src/assets/javascripts/_/index.ts | 1 + src/assets/javascripts/bundle.ts | 7 +- .../javascripts/components/toc/index.ts | 76 ++++++++++++++++--- src/assets/stylesheets/main/layout/_nav.scss | 18 ++++- 6 files changed, 118 insertions(+), 15 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 82b1a51de..6304d00bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -111,7 +111,7 @@ # "google_analytics": ["UA-XXXXX", "auto"], "globaltoc_collapse": True, "features": [ - # "navigation.expand", + "navigation.expand", # "navigation.tabs", # "toc.integrate", "navigation.sections", @@ -121,6 +121,7 @@ # "navigation.tracking", # "search.highlight", "search.share", + "toc.follow", ], "palette": [ { diff --git a/docs/customization.rst b/docs/customization.rst index c572140eb..6d54e373b 100644 --- a/docs/customization.rst +++ b/docs/customization.rst @@ -270,6 +270,34 @@ Configuration Options - `search.highlight `_ - `search.share `_ - `toc.integrate `_ + - ``toc.follow`` + + This is similar to the `toc.follow + `__ + feature supported by the mkdocs-material theme, but differs in that + both the left and right sidebar are scrolled automatically. + + - The local table-of-contents scrolls automatically to keep the + currently-displayed document section in view. Note that this + applies to all three possible locations for the local + table-of-contents: + + - in the right sidebar, which is the default location if the browser + viewport is sufficiently wide; + - in the left sidebar, if the ``toc.integrate`` feature is enabled; + - in the "layered" navigation menu, if the browser viewport is + sufficiently narrow. + + - If the ``toc.integrate`` feature is disabled, the left sidebar + additionally scrolls automatically to keep within view either: + + - the navigation entry for the current document, or + - if the current document contains sections with child documents, + the navigation entry for the currently-displayed document section. + + Note that if the ``toc.integrate`` feature is enabled, the left + sidebar is instead scrolled for the local table-of-contents as + described above. .. hint:: Sphinx automatically implements the diff --git a/src/assets/javascripts/_/index.ts b/src/assets/javascripts/_/index.ts index 2fc6e1fc3..1037f7337 100644 --- a/src/assets/javascripts/_/index.ts +++ b/src/assets/javascripts/_/index.ts @@ -45,6 +45,7 @@ export type Flag = | "search.share" /* Search sharing */ | "search.suggest" /* Search suggestions */ | "toc.integrate" /* Integrated table of contents */ + | "toc.follow" /* sphinx-immaterial: auto-scroll toc */ /* ------------------------------------------------------------------------- */ diff --git a/src/assets/javascripts/bundle.ts b/src/assets/javascripts/bundle.ts index a97c3c937..b85ac3038 100644 --- a/src/assets/javascripts/bundle.ts +++ b/src/assets/javascripts/bundle.ts @@ -220,7 +220,12 @@ const content$ = defer(() => merge( /* Table of contents */ ...getComponentElements("toc") - .map(el => mountTableOfContents(el, { viewport$, header$, target$ })), + .map(el => mountTableOfContents(el, { viewport$, header$, target$, localToc: true })), + + /* Global Table of contents */ + ...getComponentElements("sidebar") + .filter(el => el.getAttribute("data-md-type") === "navigation") + .map(el => mountTableOfContents(el, { viewport$, header$, target$, localToc: false })), /* Back-to-top button */ ...getComponentElements("top") diff --git a/src/assets/javascripts/components/toc/index.ts b/src/assets/javascripts/components/toc/index.ts index 8ecf0d8f7..9a95cdfc1 100644 --- a/src/assets/javascripts/components/toc/index.ts +++ b/src/assets/javascripts/components/toc/index.ts @@ -82,6 +82,7 @@ export interface TableOfContents { interface WatchOptions { viewport$: Observable /* Viewport observable */ header$: Observable
/* Header observable */ + excludedLinks?: Set /* sphinx-immaterial: Links to exclude */ } /** @@ -91,6 +92,7 @@ interface MountOptions { viewport$: Observable /* Viewport observable */ header$: Observable
/* Header observable */ target$: Observable /* Location target observable */ + localToc: boolean } /* ---------------------------------------------------------------------------- @@ -118,17 +120,28 @@ interface MountOptions { * @returns Table of contents observable */ export function watchTableOfContents( - el: HTMLElement, { viewport$, header$ }: WatchOptions + el: HTMLElement, { viewport$, header$, excludedLinks }: WatchOptions ): Observable { const table = new Map() /* Compute anchor-to-target mapping */ - const anchors = getElements("[href^=\\#]", el) + const anchors = getElements("a[href]", el) for (const anchor of anchors) { - const id = decodeURIComponent(anchor.hash.substring(1)) - const target = getOptionalElement(`[id="${id}"]`) - if (typeof target !== "undefined") - table.set(anchor, target) + if (excludedLinks?.has(anchor)) continue + const href = anchor.getAttribute("href")! + let target: HTMLElement|undefined + if (href.startsWith("#")) { + const id = decodeURIComponent(anchor.hash.substring(1)) + target = getOptionalElement(`[id="${id}"]`) + } else { + target = getOptionalElement(`a.pseudo-toc-entry[href=${CSS.escape(href)}]`) + } + if (typeof target !== "undefined") { + const link = anchor.closest(".md-nav__link") + if (link !== null) { + table.set(link as HTMLAnchorElement, target) + } + } } /* Compute necessary adjustment for header */ @@ -265,17 +278,19 @@ export function watchTableOfContents( * @returns Table of contents component observable */ export function mountTableOfContents( - el: HTMLElement, { viewport$, header$, target$ }: MountOptions + el: HTMLElement, { viewport$, header$, target$, localToc }: MountOptions ): Observable> { return defer(() => { const push$ = new Subject() - push$.subscribe(({ prev, next }) => { + /* sphinx-immaterial: use separate active class for local vs global toc */ + const activeClassName = localToc ? "md-nav__link--active" : "md-nav__link--in-viewport" + push$.subscribe(({ prev, next }) => { /* Look forward */ for (const [anchor] of next) { anchor.removeAttribute("data-md-state") anchor.classList.remove( - "md-nav__link--active" + activeClassName ) } @@ -283,14 +298,47 @@ export function mountTableOfContents( for (const [index, [anchor]] of prev.entries()) { anchor.setAttribute("data-md-state", "blur") anchor.classList.toggle( - "md-nav__link--active", + activeClassName, index === prev.length - 1 ) } }) + /* sphinx-immaterial: auto-scroll toc */ + if (feature("toc.follow") && (localToc || !feature("toc.integrate"))) { + let scrollToCurrentPageLinkByDefault = !localToc || feature("toc.integrate") + push$.pipe(debounceTime(1)).subscribe(({prev}) => { + let curLink: HTMLElement|undefined + if (prev.length === 0 && scrollToCurrentPageLinkByDefault) { + curLink = (el.querySelector("a[href='#']") ?? el) as HTMLElement| undefined + } + scrollToCurrentPageLinkByDefault = false + if (prev.length !== 0) { + curLink = prev[prev.length - 1][0] + } + if (curLink === undefined) return + if (!curLink.offsetHeight) return + // Find closest scrollable ancestor. + let scrollParent = curLink.parentElement + // On Firefox 101, the `scrollHeight` is sometimes 1 pixel + // larger than the `clientHeight` on non-scrollable elements. + const scrollHeightEpsilon = 5 + while (scrollParent !== null && + scrollParent.scrollHeight - scrollHeightEpsilon <= scrollParent.clientHeight) { + scrollParent = scrollParent.parentElement + } + if (scrollParent !== null && scrollParent !== document.body && + scrollParent !== document.documentElement) { + const linkRect = curLink.getBoundingClientRect() + const scrollRect = scrollParent.getBoundingClientRect() + scrollParent.scrollTo({ + top: scrollParent.scrollTop + (linkRect.y - scrollRect.height / 2 - scrollRect.y)}) + } + }) + } + /* Set up anchor tracking, if enabled */ - if (feature("navigation.tracking")) + if (localToc && feature("navigation.tracking")) viewport$ .pipe( takeUntil(push$.pipe(takeLast(1))), @@ -321,8 +369,12 @@ export function mountTableOfContents( } }) + const excludedLinks = localToc + ? undefined + : new Set(getElements("[data-md-component='toc'] a[href]", el)) + /* Create and return component */ - return watchTableOfContents(el, { viewport$, header$ }) + return watchTableOfContents(el, { viewport$, header$, excludedLinks }) .pipe( tap(state => push$.next(state)), finalize(() => push$.complete()), diff --git a/src/assets/stylesheets/main/layout/_nav.scss b/src/assets/stylesheets/main/layout/_nav.scss index b31594fdf..31958c627 100644 --- a/src/assets/stylesheets/main/layout/_nav.scss +++ b/src/assets/stylesheets/main/layout/_nav.scss @@ -103,7 +103,7 @@ scroll-snap-align: start; // Navigation link in blurred state - &[data-md-state="blur"] { + &[data-md-state="blur"]:not(&--in-viewport):not(&--active) { color: var(--md-default-fg-color--light); } @@ -112,6 +112,22 @@ color: var(--md-typeset-a-color); } + // sphinx-immaterial: show nav links corresponding to current viewport + &--in-viewport { + position: relative; + + &::before { + position: absolute; + top: 0; + right: calc(100% + px2rem(6px)); + bottom: 0; + width: px2rem(1px); + height: 100%; + background-color: var(--md-primary-fg-color); + content: ""; + } + } + // Stretch section index link to full width .md-nav__item &--index [href] { width: 100%; From 87d87f900901ec56aff1df09572f68497bafcf31 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 30 Jun 2022 19:44:30 -0700 Subject: [PATCH 17/26] Fix support for toc.integrate and non-leaf pages Previously, neither the `toc.integrate` feature, nor the "layered" TOC menu for "tablet portrait" viewports and narrower, worked correctly on non-leaf pages, due to a limitation of the upstream mkdocs-material theme. This commit avoids those limitations except on the root page. Co-authored-by: Brendan <2bndy5@gmail.com> --- sphinx_immaterial/nav_adapt.py | 189 +++++++++++++++++-- src/assets/stylesheets/main/layout/_nav.scss | 11 ++ src/partials/nav-item.html | 24 ++- src/partials/toc.html | 5 + 4 files changed, 208 insertions(+), 21 deletions(-) diff --git a/sphinx_immaterial/nav_adapt.py b/sphinx_immaterial/nav_adapt.py index 8cb9e022e..8eaa01221 100644 --- a/sphinx_immaterial/nav_adapt.py +++ b/sphinx_immaterial/nav_adapt.py @@ -1,4 +1,131 @@ -"""Injects mkdocs-style `nav` and `page` objects into the HTML jinja2 context.""" +"""Injects mkdocs-style `nav` and `page` objects into the HTML jinja2 context. + +This generates global and local tables-of-contents (TOCs) usable by (a modified +version of) the mkdocs-material HTML templates. + +In particular, for each document, this module generates three separate TOCs: + +`global_toc`: + The `global_toc` is the global table of contents. If the + `globaltoc_collapse` theme option is `False`, it contains all documents + reachable from the root document, as well as any sections of non-root + documents that contain non-empty TOCs. If the `globaltoc_collapse` theme + option is `True`, then the global TOC is restricted to children of: + + - the root, + - the current document, + - ancestors of the current document. + +`local_toc`: + The `local_toc` is the local table of contents for the specified document. + It contains all sections of the current document, but does not contain any + entries that link outside the current document. + +`integrated_local_toc`: + The `integrated_local_toc` contains all sections of the current document, as + well as any child documents referenced by a TOC contained in the current + document. Whether children of the child docuemnts are included depends on + the `globaltoc_collapse` theme option. + +Background +---------- + +The Sphinx document model differs from the mkdocs document model in that +documents can be organized as children of other documents and sections within +those documents. + +Similar functionality is optionally available in mkdocs-material, through the +`navigation.indexes` feature, which effectively allows documents to be children +of other documents (but not sections within those documents). + +However, mkdocs-material specifically documents that `navigation.indexes` is +incompatible with the `toc.integrate` feature. Furthermore, as noted in +https://github.com/squidfunk/mkdocs-material/issues/3819, the local TOC is +inaccessible in the layered navigation menu used with narrow viewports. This is +because under the mkdocs-material document model (with `navigation.indexes` +feature enabled), there is no natural way to combine both the local TOC for a +page and the nested list of child documents into a single TOC. + +With Sphinx, non-leaf documents are the common case, not a special added +feature, and it is not very reasonable for the TOC to not behave correctly on +such documents. Furthermore, under the Sphinx document model, child documents +are already organized within the sections of their parent document. Therefore, +there *is* a natural way to display the local TOC and the nested child documents +as a single TOC --- this combined toc is the `integrated_local_toc`. + +The mkdocs-material package uses the global and local TOCs as follows: + +Left sidebar: +- Doc 1 (from global_toc) +- Group (from global_toc) + - Doc 2 (from global_toc) + - Current page (from global_toc) + - Section 1 (from local_toc) + - Section 2 (from local_toc) + +Right side bar: +- Section 1 (from local_toc) +- Section 2 (from local_toc) + +Note that the local TOC is duplicated into the left sidebar as well, but is +hidden in the normal non-mobile layout, unless the `toc.integrate` feature is +enabled (in that case the right side bar is always hidden). With a sufficiently +narrow layout, the right side bar is hidden and the duplicate copy of the local +toc in the left sidebar is shown in the layered "hamburger" navigation menu. + +The above example is for the case where the current page is a leaf page. If the +`navigation.indexes` feature is in use and the current page is a non-leaf page, +the sidebars are instead generated as follows: + +Left sidebar: +- Doc 1 (from global_toc) +- Group (from global_toc) + - Doc 2 (from global_toc) + - Current page (from global_toc) + - Doc 3 (from global_toc) + - Doc 4 (from global_toc) + +Right side bar: +- Section 1 (from local_toc) +- Section 2 (from local_toc) + +In order to support a separate `integrated_local_toc`, this theme modifies the +mkdocs-material templates to generate the sidebars as follows: + +Left sidebar: +- Doc 1 (from global_toc) +- Group (from global_toc) + - Doc 2 (from global_toc) + - Current page (from global_toc) [class=md-nav__current-nested] + - Doc 3 (from global_toc) + - Doc 4 (from global_toc) + - Current page (from global_toc) [class=md-nav__current-toc] + - Section 1 (from local_toc_integrated) + - Section 2 (from local_toc_integrated) + +Right side bar: +- Section 1 (from local_toc) +- Section 2 (from local_toc) + +The left sidebar contains two copies of the local toc, one generated from +`global_toc` and the other from `local_toc_integrated`, but CSS rules based on +the added `md-nav__current-nested` and `md-nav__current-toc` ensure that at most +one copy is shown at a time. + +The root document is an exception: in Sphinx the global document structure is +defined by adding `toctree` nodes to the root document. Technically those +`toctree` nodes are still contained within the usual section structure of the +root document, but the built-in TOC functionality in Sphinx treats the root +document specially, and extacts any `toctree` nodes into a separate global TOC +hierarchy, independent of the section structure of the root document. In +practice, users often place the `toctree` nodes at the end of the root document, +effectively making them children of the last section, but it is not intended +that they are actually a part of any section. Therefore, for the root document +there is no natural way to integrate the local and global TOCs, and consequently +the local TOC is simply unavailable when the `toc.integrate` feature is enabled +or when using the "layered" navigation menu. + +""" import collections import copy @@ -69,7 +196,7 @@ class MkdocsNavEntry: # Excludes links to sections within in an active page. active: bool # Set to `True` if this page is the current page. Excludes links to - # sections within in an active page. + # sections within an active page. current: bool # Set to `True` if `active`, or if this is a link to a section within an `active` page. @@ -486,7 +613,7 @@ def _make_toc_for_page(key: TocEntryKey, children: List[MkdocsNavEntry]): child.url = sphinx.util.osutil.relative_uri( real_page_url, root_relative_url ) - if uri.fragment: + if uri.fragment or child.url == "": child.url += f"#{uri.fragment}" in_ancestors = child_key in ancestors child_active = False @@ -508,8 +635,19 @@ def _make_toc_for_page(key: TocEntryKey, children: List[MkdocsNavEntry]): def _get_mkdocs_tocs( - app: sphinx.application.Sphinx, pagename: str, duplicate_local_toc: bool -) -> Tuple[List[MkdocsNavEntry], List[MkdocsNavEntry]]: + app: sphinx.application.Sphinx, + pagename: str, + duplicate_local_toc: bool, + toc_integrate: bool, +) -> Tuple[List[MkdocsNavEntry], List[MkdocsNavEntry], List[MkdocsNavEntry]]: + """Generates the global and local TOCs for a document. + + :param app: The sphinx application object. + :param pagename: The name of the document for which to generate the tocs. + :param duplicate_local_toc: Duplicate the local toc in the global toc. + :param toc_integrate: Indicates if the `toc.integrate` feature is enabled. + :returns: A tuple `(global_toc, local_toc, integrated_local_toc)`. + """ theme_options = app.config["html_theme_options"] global_toc = _get_global_toc( app=app, @@ -517,6 +655,7 @@ def _get_mkdocs_tocs( collapse=theme_options.get("globaltoc_collapse", False), ) local_toc: List[MkdocsNavEntry] = [] + integrated_local_toc: List[MkdocsNavEntry] = [] env = app.env assert env is not None builder = app.builder @@ -524,12 +663,17 @@ def _get_mkdocs_tocs( if pagename != env.config.master_doc: # Extract entry from `global_toc` corresponding to the current page. current_page_toc_entry = _get_current_page_in_toc(global_toc) - if current_page_toc_entry: - local_toc = cast( - List[MkdocsNavEntry], - [_prune_toc_by_active(current_page_toc_entry, active=True)], - ) - if not duplicate_local_toc: + if current_page_toc_entry is not None: + integrated_local_toc = [copy.copy(current_page_toc_entry)] + integrated_local_toc[0].children = list(integrated_local_toc[0].children) + if not toc_integrate: + local_toc = cast( + List[MkdocsNavEntry], + [_prune_toc_by_active(current_page_toc_entry, active=True)], + ) + if toc_integrate: + current_page_toc_entry.children = [] + elif not duplicate_local_toc: current_page_toc_entry.children = [ child for child in [ @@ -538,21 +682,20 @@ def _get_mkdocs_tocs( ] if child is not None ] - else: # Every page is a child of the root page. We still want a full TOC # tree, though. - local_toc_node = sphinx.environment.adapters.toctree.TocTree(env).get_toc_for( - pagename, - builder, - ) + local_toc_node = env.tocs[pagename] local_toc = _get_mkdocs_toc(local_toc_node, builder) _add_domain_info_to_toc(app, local_toc, pagename) if len(local_toc) == 1 and len(local_toc[0].children) == 0: local_toc = [] - return global_toc, local_toc + if len(integrated_local_toc) == 1 and len(integrated_local_toc[0].children) == 0: + integrated_local_toc = [] + + return global_toc, local_toc, integrated_local_toc def _html_page_context( @@ -566,16 +709,19 @@ def _html_page_context( assert env is not None theme_options: dict = app.config["html_theme_options"] + features = theme_options.get("features", ()) + assert isinstance(features, collections.abc.Sequence) page_title = markupsafe.Markup.escape( markupsafe.Markup(context.get("title")).striptags() ) meta = context.get("meta") if meta is None: meta = {} - global_toc, local_toc = _get_mkdocs_tocs( + global_toc, local_toc, integrated_local_toc = _get_mkdocs_tocs( app, pagename, duplicate_local_toc=isinstance(meta.get("duplicate-local-toc"), str), + toc_integrate="toc.integrate" in features, ) context.update(nav=_NavContextObject(global_toc)) context["nav"].homepage = dict( @@ -607,11 +753,18 @@ def _html_page_context( # the local toc. local_toc = local_toc[0].children + if len(integrated_local_toc) == 1: + # If there is a single top-level heading, it is treated as the page + # heading, and it would be redundant to also include it as an entry in + # the local toc. + integrated_local_toc = integrated_local_toc[0].children + # Add other context values in mkdocs/mkdocs-material format. page = dict( title=page_title, is_homepage=(pagename == context["master_doc"]), toc=local_toc, + integrated_local_toc=integrated_local_toc, meta={"hide": [], "revision_date": context.get("last_updated")}, content=context.get("body"), ) diff --git a/src/assets/stylesheets/main/layout/_nav.scss b/src/assets/stylesheets/main/layout/_nav.scss index 31958c627..8c1bad7fb 100644 --- a/src/assets/stylesheets/main/layout/_nav.scss +++ b/src/assets/stylesheets/main/layout/_nav.scss @@ -421,6 +421,12 @@ // [tablet portrait -]: Layered navigation with table of contents @include break-to-device(tablet portrait) { + // sphinx-immaterial: hide nested nav items of current page, since + // they are redundant with integrated toc. + &__current-nested { + display: none; + } + // Show link to table of contents &--primary &__link[for="__toc"] { display: flex; @@ -477,6 +483,11 @@ // [tablet landscape +]: Tree-like table of contents @include break-from-device(tablet landscape) { + // sphinx-immaterial: hide integreated toc, since it is redundant with any nested items. + &__current-toc { + display: none; + } + // Navigation title &--secondary &__title { diff --git a/src/partials/nav-item.html b/src/partials/nav-item.html index 05e357eb6..0c81176e1 100644 --- a/src/partials/nav-item.html +++ b/src/partials/nav-item.html @@ -29,6 +29,12 @@ {% set class = class ~ " md-nav__item--active" %} {% endif %} + + {%- set orig_class = class %} + {%- if nav_item.current %} + {% set class = class ~ " md-nav__current-nested" %} + {%- endif %} + {% if nav_item.children %} @@ -111,9 +117,20 @@ - {% elif nav_item.active %} + + {% endif %} + {% if nav_item.current %} + + + {%- set class = orig_class %} + {%- if nav_item.children %} + {% set class = class ~ " md-nav__current-toc" %} + {%- endif %} +
  • - {% set toc = page.toc %} + + {% set toc = page.integrated_local_toc %} + {% set use_integrated_local_toc = True %} - {% else %} + + {% elif not nav_item.children %}
  • {{ nav_item.title }} diff --git a/src/partials/toc.html b/src/partials/toc.html index 019454c9f..2f4717a81 100644 --- a/src/partials/toc.html +++ b/src/partials/toc.html @@ -30,6 +30,11 @@