"""Generates reST source files for autosummary. Usable as a library or script to generate automatic RST source files for items referred to in autosummary:: directives. Each generated RST file contains a single auto*:: directive which extracts the docstring of the referred item. Example Makefile rule:: generate: sphinx-autogen -o source/generated source/*.rst """ from __future__ import annotations import argparse import importlib import inspect import locale import os import pkgutil import pydoc import re import sys from os import path from typing import TYPE_CHECKING, Any, NamedTuple from jinja2 import TemplateNotFound from jinja2.sandbox import SandboxedEnvironment import sphinx.locale from sphinx import __display_version__, package_dir from sphinx.builders import Builder from sphinx.config import Config from sphinx.ext.autodoc.importer import import_module from sphinx.ext.autosummary import ( ImportExceptionGroup, get_documenter, import_by_name, import_ivar_by_name, ) from sphinx.locale import __ from sphinx.pycode import ModuleAnalyzer, PycodeError from sphinx.registry import SphinxComponentRegistry from sphinx.util import logging, rst from sphinx.util.inspect import getall, safe_getattr from sphinx.util.osutil import ensuredir from sphinx.util.template import SphinxTemplateLoader if TYPE_CHECKING: from collections.abc import Sequence, Set from gettext import NullTranslations from sphinx.application import Sphinx from sphinx.ext.autodoc import Documenter logger = logging.getLogger(__name__) class DummyApplication: """Dummy Application class for sphinx-autogen command.""" def __init__(self, translator: NullTranslations) -> None: self.config = Config() self.registry = SphinxComponentRegistry() self.messagelog: list[str] = [] self.srcdir = "/" self.translator = translator self.verbosity = 0 self._warncount = 0 self.warningiserror = False self.config.add('autosummary_context', {}, True, None) self.config.add('autosummary_filename_map', {}, True, None) self.config.add('autosummary_ignore_module_all', True, 'env', bool) self.config.init_values() def emit_firstresult(self, *args: Any) -> None: pass class AutosummaryEntry(NamedTuple): name: str path: str | None template: str recursive: bool def setup_documenters(app: Any) -> None: from sphinx.ext.autodoc import ( AttributeDocumenter, ClassDocumenter, DataDocumenter, DecoratorDocumenter, ExceptionDocumenter, FunctionDocumenter, MethodDocumenter, ModuleDocumenter, PropertyDocumenter, ) documenters: list[type[Documenter]] = [ ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter, FunctionDocumenter, MethodDocumenter, AttributeDocumenter, DecoratorDocumenter, PropertyDocumenter, ] for documenter in documenters: app.registry.add_documenter(documenter.objtype, documenter) def _underline(title: str, line: str = '=') -> str: if '\n' in title: msg = 'Can only underline single lines' raise ValueError(msg) return title + '\n' + line * len(title) class AutosummaryRenderer: """A helper class for rendering.""" def __init__(self, app: Sphinx) -> None: if isinstance(app, Builder): msg = 'Expected a Sphinx application object!' raise ValueError(msg) system_templates_path = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')] loader = SphinxTemplateLoader(app.srcdir, app.config.templates_path, system_templates_path) self.env = SandboxedEnvironment(loader=loader) self.env.filters['escape'] = rst.escape self.env.filters['e'] = rst.escape self.env.filters['underline'] = _underline if app.translator: self.env.add_extension("jinja2.ext.i18n") self.env.install_gettext_translations(app.translator) def render(self, template_name: str, context: dict) -> str: """Render a template file.""" try: template = self.env.get_template(template_name) except TemplateNotFound: try: # objtype is given as template_name template = self.env.get_template('autosummary/%s.rst' % template_name) except TemplateNotFound: # fallback to base.rst template = self.env.get_template('autosummary/base.rst') return template.render(context) def _split_full_qualified_name(name: str) -> tuple[str | None, str]: """Split full qualified name to a pair of modname and qualname. A qualname is an abbreviation for "Qualified name" introduced at PEP-3155 (https://peps.python.org/pep-3155/). It is a dotted path name from the module top-level. A "full" qualified name means a string containing both module name and qualified name. .. note:: This function actually imports the module to check its existence. Therefore you need to mock 3rd party modules if needed before calling this function. """ parts = name.split('.') for i, _part in enumerate(parts, 1): try: modname = ".".join(parts[:i]) importlib.import_module(modname) except ImportError: if parts[:i - 1]: return ".".join(parts[:i - 1]), ".".join(parts[i - 1:]) else: return None, ".".join(parts) except IndexError: pass return name, "" # -- Generating output --------------------------------------------------------- class ModuleScanner: def __init__(self, app: Any, obj: Any) -> None: self.app = app self.object = obj def get_object_type(self, name: str, value: Any) -> str: return get_documenter(self.app, value, self.object).objtype def is_skipped(self, name: str, value: Any, objtype: str) -> bool: try: return self.app.emit_firstresult('autodoc-skip-member', objtype, name, value, False, {}) except Exception as exc: logger.warning(__('autosummary: failed to determine %r to be documented, ' 'the following exception was raised:\n%s'), name, exc, type='autosummary') return False def scan(self, imported_members: bool) -> list[str]: members = [] try: analyzer = ModuleAnalyzer.for_module(self.object.__name__) attr_docs = analyzer.find_attr_docs() except PycodeError: attr_docs = {} for name in members_of(self.object, self.app.config): try: value = safe_getattr(self.object, name) except AttributeError: value = None objtype = self.get_object_type(name, value) if self.is_skipped(name, value, objtype): continue try: if ('', name) in attr_docs: imported = False elif inspect.ismodule(value): # NoQA: SIM114 imported = True elif safe_getattr(value, '__module__') != self.object.__name__: imported = True else: imported = False except AttributeError: imported = False respect_module_all = not self.app.config.autosummary_ignore_module_all if ( # list all members up imported_members # list not-imported members or imported is False # list members that have __all__ set or (respect_module_all and '__all__' in dir(self.object)) ): members.append(name) return members def members_of(obj: Any, conf: Config) -> Sequence[str]: """Get the members of ``obj``, possibly ignoring the ``__all__`` module attribute Follows the ``conf.autosummary_ignore_module_all`` setting.""" if conf.autosummary_ignore_module_all: return dir(obj) else: return getall(obj) or dir(obj) def generate_autosummary_content(name: str, obj: Any, parent: Any, template: AutosummaryRenderer, template_name: str, imported_members: bool, app: Any, recursive: bool, context: dict, modname: str | None = None, qualname: str | None = None) -> str: doc = get_documenter(app, obj, parent) ns: dict[str, Any] = {} ns.update(context) if doc.objtype == 'module': scanner = ModuleScanner(app, obj) ns['members'] = scanner.scan(imported_members) respect_module_all = not app.config.autosummary_ignore_module_all imported_members = imported_members or ('__all__' in dir(obj) and respect_module_all) ns['functions'], ns['all_functions'] = \ _get_members(doc, app, obj, {'function'}, imported=imported_members) ns['classes'], ns['all_classes'] = \ _get_members(doc, app, obj, {'class'}, imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ _get_members(doc, app, obj, {'exception'}, imported=imported_members) ns['attributes'], ns['all_attributes'] = \ _get_module_attrs(name, ns['members']) ispackage = hasattr(obj, '__path__') if ispackage and recursive: # Use members that are not modules as skip list, because it would then mean # that module was overwritten in the package namespace skip = ( ns["all_functions"] + ns["all_classes"] + ns["all_exceptions"] + ns["all_attributes"] ) # If respect_module_all and module has a __all__ attribute, first get # modules that were explicitly imported. Next, find the rest with the # get_modules method, but only put in "public" modules that are in the # __all__ list # # Otherwise, use get_modules method normally if respect_module_all and '__all__' in dir(obj): imported_modules, all_imported_modules = \ _get_members(doc, app, obj, {'module'}, imported=True) skip += all_imported_modules imported_modules = [name + '.' + modname for modname in imported_modules] all_imported_modules = \ [name + '.' + modname for modname in all_imported_modules] public_members = getall(obj) else: imported_modules, all_imported_modules = [], [] public_members = None modules, all_modules = _get_modules(obj, skip=skip, name=name, public_members=public_members) ns['modules'] = imported_modules + modules ns["all_modules"] = all_imported_modules + all_modules elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ set(dir(obj)) - set(obj.__dict__.keys()) ns['methods'], ns['all_methods'] = \ _get_members(doc, app, obj, {'method'}, include_public={'__init__'}) ns['attributes'], ns['all_attributes'] = \ _get_members(doc, app, obj, {'attribute', 'property'}) if modname is None or qualname is None: modname, qualname = _split_full_qualified_name(name) if doc.objtype in ('method', 'attribute', 'property'): ns['class'] = qualname.rsplit(".", 1)[0] if doc.objtype in ('class',): shortname = qualname else: shortname = qualname.rsplit(".", 1)[-1] ns['fullname'] = name ns['module'] = modname ns['objname'] = qualname ns['name'] = shortname ns['objtype'] = doc.objtype ns['underline'] = len(name) * '=' if template_name: return template.render(template_name, ns) else: return template.render(doc.objtype, ns) def _skip_member(app: Sphinx, obj: Any, name: str, objtype: str) -> bool: try: return app.emit_firstresult('autodoc-skip-member', objtype, name, obj, False, {}) except Exception as exc: logger.warning(__('autosummary: failed to determine %r to be documented, ' 'the following exception was raised:\n%s'), name, exc, type='autosummary') return False def _get_class_members(obj: Any) -> dict[str, Any]: members = sphinx.ext.autodoc.get_class_members(obj, None, safe_getattr) return {name: member.object for name, member in members.items()} def _get_module_members(app: Sphinx, obj: Any) -> dict[str, Any]: members = {} for name in members_of(obj, app.config): try: members[name] = safe_getattr(obj, name) except AttributeError: continue return members def _get_all_members(doc: type[Documenter], app: Sphinx, obj: Any) -> dict[str, Any]: if doc.objtype == 'module': return _get_module_members(app, obj) elif doc.objtype == 'class': return _get_class_members(obj) return {} def _get_members(doc: type[Documenter], app: Sphinx, obj: Any, types: set[str], *, include_public: Set[str] = frozenset(), imported: bool = True) -> tuple[list[str], list[str]]: items: list[str] = [] public: list[str] = [] all_members = _get_all_members(doc, app, obj) for name, value in all_members.items(): documenter = get_documenter(app, value, obj) if documenter.objtype in types: # skip imported members if expected if imported or getattr(value, '__module__', None) == obj.__name__: skipped = _skip_member(app, value, name, documenter.objtype) if skipped is True: pass elif skipped is False: # show the member forcedly items.append(name) public.append(name) else: items.append(name) if name in include_public or not name.startswith('_'): # considers member as public public.append(name) return public, items def _get_module_attrs(name: str, members: Any) -> tuple[list[str], list[str]]: """Find module attributes with docstrings.""" attrs, public = [], [] try: analyzer = ModuleAnalyzer.for_module(name) attr_docs = analyzer.find_attr_docs() for namespace, attr_name in attr_docs: if namespace == '' and attr_name in members: attrs.append(attr_name) if not attr_name.startswith('_'): public.append(attr_name) except PycodeError: pass # give up if ModuleAnalyzer fails to parse code return public, attrs def _get_modules( obj: Any, *, skip: Sequence[str], name: str, public_members: Sequence[str] | None = None) -> tuple[list[str], list[str]]: items: list[str] = [] public: list[str] = [] for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__): if modname in skip: # module was overwritten in __init__.py, so not accessible continue fullname = name + '.' + modname try: module = import_module(fullname) if module and hasattr(module, '__sphinx_mock__'): continue except ImportError: pass items.append(fullname) if public_members is not None: if modname in public_members: public.append(fullname) else: if not modname.startswith('_'): public.append(fullname) return public, items def generate_autosummary_docs(sources: list[str], output_dir: str | os.PathLike[str] | None = None, suffix: str = '.rst', base_path: str | os.PathLike[str] | None = None, imported_members: bool = False, app: Any = None, overwrite: bool = True, encoding: str = 'utf-8') -> None: showed_sources = sorted(sources) if len(showed_sources) > 20: showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:] logger.info(__('[autosummary] generating autosummary for: %s') % ', '.join(showed_sources)) if output_dir: logger.info(__('[autosummary] writing to %s') % output_dir) if base_path is not None: sources = [os.path.join(base_path, filename) for filename in sources] template = AutosummaryRenderer(app) # read items = find_autosummary_in_files(sources) # keep track of new files new_files = [] if app: filename_map = app.config.autosummary_filename_map else: filename_map = {} # write for entry in sorted(set(items), key=str): if entry.path is None: # The corresponding autosummary:: directive did not have # a :toctree: option continue path = output_dir or os.path.abspath(entry.path) ensuredir(path) try: name, obj, parent, modname = import_by_name(entry.name) qualname = name.replace(modname + ".", "") except ImportExceptionGroup as exc: try: # try to import as an instance attribute name, obj, parent, modname = import_ivar_by_name(entry.name) qualname = name.replace(modname + ".", "") except ImportError as exc2: if exc2.__cause__: exceptions: list[BaseException] = exc.exceptions + [exc2.__cause__] else: exceptions = exc.exceptions + [exc2] errors = list({f"* {type(e).__name__}: {e}" for e in exceptions}) logger.warning(__('[autosummary] failed to import %s.\nPossible hints:\n%s'), entry.name, '\n'.join(errors)) continue context: dict[str, Any] = {} if app: context.update(app.config.autosummary_context) content = generate_autosummary_content(name, obj, parent, template, entry.template, imported_members, app, entry.recursive, context, modname, qualname) filename = os.path.join(path, filename_map.get(name, name) + suffix) if os.path.isfile(filename): with open(filename, encoding=encoding) as f: old_content = f.read() if content == old_content: continue if overwrite: # content has changed with open(filename, 'w', encoding=encoding) as f: f.write(content) new_files.append(filename) else: with open(filename, 'w', encoding=encoding) as f: f.write(content) new_files.append(filename) # descend recursively to new files if new_files: generate_autosummary_docs(new_files, output_dir=output_dir, suffix=suffix, base_path=base_path, imported_members=imported_members, app=app, overwrite=overwrite) # -- Finding documented entries in files --------------------------------------- def find_autosummary_in_files(filenames: list[str]) -> list[AutosummaryEntry]: """Find out what items are documented in source/*.rst. See `find_autosummary_in_lines`. """ documented: list[AutosummaryEntry] = [] for filename in filenames: with open(filename, encoding='utf-8', errors='ignore') as f: lines = f.read().splitlines() documented.extend(find_autosummary_in_lines(lines, filename=filename)) return documented def find_autosummary_in_docstring( name: str, filename: str | None = None, ) -> list[AutosummaryEntry]: """Find out what items are documented in the given object's docstring. See `find_autosummary_in_lines`. """ try: real_name, obj, parent, modname = import_by_name(name) lines = pydoc.getdoc(obj).splitlines() return find_autosummary_in_lines(lines, module=name, filename=filename) except AttributeError: pass except ImportExceptionGroup as exc: errors = '\n'.join({f"* {type(e).__name__}: {e}" for e in exc.exceptions}) logger.warning(f'Failed to import {name}.\nPossible hints:\n{errors}') # NoQA: G004 except SystemExit: logger.warning("Failed to import '%s'; the module executes module level " 'statement and it might call sys.exit().', name) return [] def find_autosummary_in_lines( lines: list[str], module: str | None = None, filename: str | None = None, ) -> list[AutosummaryEntry]: """Find out what items appear in autosummary:: directives in the given lines. Returns a list of (name, toctree, template) where *name* is a name of an object and *toctree* the :toctree: path of the corresponding autosummary directive (relative to the root of the file name), and *template* the value of the :template: option. *toctree* and *template* ``None`` if the directive does not have the corresponding options set. """ autosummary_re = re.compile(r'^(\s*)\.\.\s+autosummary::\s*') automodule_re = re.compile( r'^\s*\.\.\s+automodule::\s*([A-Za-z0-9_.]+)\s*$') module_re = re.compile( r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$') autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?') recursive_arg_re = re.compile(r'^\s+:recursive:\s*$') toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$') template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$') documented: list[AutosummaryEntry] = [] recursive = False toctree: str | None = None template = '' current_module = module in_autosummary = False base_indent = "" for line in lines: if in_autosummary: m = recursive_arg_re.match(line) if m: recursive = True continue m = toctree_arg_re.match(line) if m: toctree = m.group(1) if filename: toctree = os.path.join(os.path.dirname(filename), toctree) continue m = template_arg_re.match(line) if m: template = m.group(1).strip() continue if line.strip().startswith(':'): continue # skip options m = autosummary_item_re.match(line) if m: name = m.group(1).strip() if name.startswith('~'): name = name[1:] if current_module and \ not name.startswith(current_module + '.'): name = f"{current_module}.{name}" documented.append(AutosummaryEntry(name, toctree, template, recursive)) continue if not line.strip() or line.startswith(base_indent + " "): continue in_autosummary = False m = autosummary_re.match(line) if m: in_autosummary = True base_indent = m.group(1) recursive = False toctree = None template = '' continue m = automodule_re.search(line) if m: current_module = m.group(1).strip() # recurse into the automodule docstring documented.extend(find_autosummary_in_docstring( current_module, filename=filename)) continue m = module_re.match(line) if m: current_module = m.group(2) continue return documented def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( usage='%(prog)s [OPTIONS] ...', epilog=__('For more information, visit .'), description=__(""" Generate ReStructuredText using autosummary directives. sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates the reStructuredText files from the autosummary directives contained in the given input files. The format of the autosummary directive is documented in the ``sphinx.ext.autosummary`` Python module and can be read using:: pydoc sphinx.ext.autosummary """)) parser.add_argument('--version', action='version', dest='show_version', version='%%(prog)s %s' % __display_version__) parser.add_argument('source_file', nargs='+', help=__('source files to generate rST files for')) parser.add_argument('-o', '--output-dir', action='store', dest='output_dir', help=__('directory to place all output in')) parser.add_argument('-s', '--suffix', action='store', dest='suffix', default='rst', help=__('default suffix for files (default: ' '%(default)s)')) parser.add_argument('-t', '--templates', action='store', dest='templates', default=None, help=__('custom template directory (default: ' '%(default)s)')) parser.add_argument('-i', '--imported-members', action='store_true', dest='imported_members', default=False, help=__('document imported members (default: ' '%(default)s)')) parser.add_argument('-a', '--respect-module-all', action='store_true', dest='respect_module_all', default=False, help=__('document exactly the members in module __all__ attribute. ' '(default: %(default)s)')) return parser def main(argv: Sequence[str] = (), /) -> None: locale.setlocale(locale.LC_ALL, '') sphinx.locale.init_console() app = DummyApplication(sphinx.locale.get_translator()) logging.setup(app, sys.stdout, sys.stderr) # type: ignore[arg-type] setup_documenters(app) args = get_parser().parse_args(argv or sys.argv[1:]) if args.templates: app.config.templates_path.append(path.abspath(args.templates)) app.config.autosummary_ignore_module_all = ( # type: ignore[attr-defined] not args.respect_module_all ) generate_autosummary_docs(args.source_file, args.output_dir, '.' + args.suffix, imported_members=args.imported_members, app=app) if __name__ == '__main__': main(sys.argv[1:])