Implement cli and config switches for breadcrumbs and indexes

* Add command line and config switch to toggle breadcrumbs and indexes
  in antsibull-docs collection, current, devel, and stable subommands
  * This integrates changes from felixfontein in:
    #288
  * Create another mixin arg parser for options which toggle whole-site
    related content (breadcrumbs, indexes).  whole-site features control
    creation of docsite-like sites.  non-whole-site features deal with
    individual collections.
  * Comment each of the cli parsers created by stable.py and how they
    are used.
* Config:
  * Alphabetize the fields
  * Add breadcrumbs and indexes to the config file
  * Add the new switches to the sample config file
* Context:
  * Alphabetize the field names in _FIELDS_IN_APP_CTX and _FIELDS_IN_LIB_CTX
  * Add breadcrumbs and index to the context.
  * Update documentation on how to specify a cli flag in argparse so
    that it works with the context implementation.
* Remove hardcoded ADD_TOCTREES setting in favor of the breadcrumbs
  flag.
* Rename add_toctrees and tocrees parameters to breadcrumbs throughout
  the codebase.
* Unrelated: correct rst `.. warning::` roles from `.. warn::`

antsibull.cfg

@@ -1,7 +1,9 @@
 ansible_base_url = https://github.com/ansible/ansible/
+breadcrumbs = True
 doc_parsing_backend = ansible-internal
 chunksize = 4096
 galaxy_url = https://galaxy.ansible.com/
+indexes = True
 process_max = none
 pypi_url = https://pypi.org/
 thread_max = 80

antsibull/app_context.py

@@ -18,7 +18,7 @@
 This is due to limitations in the backport of the contextvars library to Python3.6.  For code which
 targets Python3.7 and above, there is no such limitation.
 
-.. warn:: This is not a stable interface.
+.. warning:: This is not a stable interface.
 
     The API has quite a few rough edges that need to be ironed out before this is finished.  Some of
     this code and data will be moved into an antibull.context module which can deal with the generic
@@ -69,12 +69,13 @@ def run(args):
     import aiocontextvars  # noqa: F401
 
 
-#: Field names in the args and config which whose value will be added to the app_ctx
-_FIELDS_IN_APP_CTX = frozenset(('galaxy_url', 'logging_cfg', 'pypi_url'))
+#: Field names in the args and config whose value will be added to the app_ctx
+_FIELDS_IN_APP_CTX = frozenset(('ansible_base_url', 'breadcrumbs', 'galaxy_url', 'indexes',
+                                'logging_cfg', 'pypi_url'))
 
-#: Field names in the args and config which whose value will be added to the lib_ctx
+#: Field names in the args and config whose value will be added to the lib_ctx
 _FIELDS_IN_LIB_CTX = frozenset(
-    ('chunksize', 'process_max', 'thread_max', 'max_retries', 'doc_parsing_backend'))
+    ('chunksize', 'doc_parsing_backend', 'max_retries', 'process_max', 'thread_max'))
 
 #: lib_ctx should be restricted to things which do not belong in the API but an application or
 #: user might want to tweak.  Global, internal, incidental values are good to store here.  Things
@@ -103,7 +104,7 @@ def run(args):
 #: a single place and then consult them globally.  The values should be passed explicitly from
 #: the application code to the library code as a function parameter.
 #:
-#: If the library provides several function to retrieve different pieces of information from the
+#: If the library provides several functions to retrieve different pieces of information from the
 #: server, the library can provide a class which takes the server's URL as a parameter and stores
 #: as an attribute and the functions can be converted into methods of the object.  Then the
 #: application code can initialize the object once and thereafter call the object's methods.
@@ -215,20 +216,39 @@ class AppContext(BaseModel):
         values stored in extras need default values, they need to be set outside of the context
         or the entries can be given an actual entry in the AppContext to take advantage of the
         schema's checking, normalization, and default setting.
+    :ivar ansible_base_url: Url to the ansible-core git repo.
+    :ivar breadcrumbs: If True, build with breadcrumbs on the plugin pages (this takes more memory).
     :ivar galaxy_url: URL of the galaxy server to get collection info from
+    :ivar indexes: If True, create index pages for all collections and all plugins in a collection.
     :ivar logging_cfg: Configuration of the application logging
-    :ivar pypi_url: URL of thepypi server to query for information
+    :ivar pypi_url: URL of the pypi server to query for information
     """
 
     extra: ContextDict = ContextDict()
     # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684
     ansible_base_url: p.HttpUrl = 'https://github.com/ansible/ansible/'
+    breadcrumbs: p.StrictBool = True
     # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684
     galaxy_url: p.HttpUrl = 'https://galaxy.ansible.com/'
+    indexes: p.StrictBool = True
     logging_cfg: LoggingModel = LoggingModel.parse_obj(DEFAULT_LOGGING_CONFIG)
     # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684
     pypi_url: p.HttpUrl = 'https://pypi.org/'
 
+    @p.validator('breadcrumbs', 'indexes', pre=True)
+    def convert_to_bool(cls, value):
+        if isinstance(value, str):
+            if value.lower() in ('0', 'false', 'no', 'n', 'f', ''):
+                value = False
+            else:
+                value = True
+        elif isinstance(value, int):
+            if value == 0:
+                value = False
+            else:
+                value = True
+        return value
+
 
 class LibContext(BaseModel):
     """
@@ -237,13 +257,18 @@ class LibContext(BaseModel):
     :ivar chunksize: number of bytes to read or write at one time for network or file IO
     :ivar process_max: Maximum number of worker processes for parallel operations
     :ivar thread_max: Maximum number of helper threads for parallel operations
+    :ivar max_retries: Maximum number of times to retry an http request (in case of timeouts and
+        other transient problems.
+    :ivar doc_parsing_backend: The backend to use for parsing the documentation strings from
+        plugins.  'ansible-internal' is the fastest.  'ansible-doc' exists in case of problems with
+        the ansible-internal backend.
     """
 
     chunksize: int = 4096
+    doc_parsing_backend: str = 'ansible-internal'
+    max_retries: int = 10
     process_max: t.Optional[int] = None
     thread_max: int = 64
-    max_retries: int = 10
-    doc_parsing_backend: str = 'ansible-internal'
 
     @p.validator('process_max', pre=True)
     def convert_to_none(cls, value):
@@ -316,16 +341,24 @@ def create_contexts(args: t.Optional[argparse.Namespace] = None,
     the context.  It validates, normalizes, and sets defaults for the contexts based on what is
     available in the arguments and configuration.
 
-    :kwarg args: An :python:obj:`argparse.Namespace` holding the program's command line arguments.
-        Note argparse's ability to add default values should not be used with fields which are fully
-        expressed in the :obj:`AppContext` or :obj:`LibContext` models.  Instead, set a default in
-        the context model.  You can use argpase defaults with fields that get set in
-        :attr:`AppContext.extra`.
+    :kwarg args: An :python:obj:`argparse.Namespace` holding the program's command line
+        arguments.  See the warning below about working with :python:mod:`argpase`.
     :kwarg cfg: A dictionary holding the program's configuration.
     :kwarg use_extra: When True, the default, all extra arguments and config values will be set as
         fields in ``app_ctx.extra``.  When False, the extra arguments and config values will be
         returned as part of the ContextReturn.
     :returns: A ContextReturn NamedTuple.
+
+    .. warning::
+        We cannot tell whether a user set a value via the command line if :python:mod:`argparse`
+        sets the field to a default value.  That means when you specify the field in the
+        :obj:`AppContext` or :obj:`LibContext` models, you must tell :python:mod:`argparse` not to
+        set the field to a default like this::
+
+            parser.add_argument('--breadcrumbs', default=argparse.SUPPRESS)
+
+        If the field is only used via the :attr:`AppContext.extra` mechanism (not explictly set),
+        then you should ignore this section and use :python:mod:`argparse`'s default mechanism.
     """
     lib_values = _extract_lib_context_values(args, cfg)
     app_values = _extract_app_context_values(args, cfg)

antsibull/augment_docs.py

@@ -23,7 +23,7 @@ def add_full_key(options_data: t.Mapping[str, t.Any], suboption_entry: str,
     :kwarg _full_key: This is a recursive function.  After we pass the first level of nesting,
         ``_full_key`` is set to record the names of the upper levels of the hierarchy.
 
-    .. warn:: This function operates by side-effect.  The options_data dictionay is modified
+    .. warning:: This function operates by side-effect.  The options_data dictionay is modified
         directly.
     """
     if _full_key is None:
@@ -51,7 +51,7 @@ def augment_docs(plugin_info: t.MutableMapping[str, t.MutableMapping[str, t.Any]
 
     :arg plugin_info: The plugin_info that will be analyzed and augmented.
 
-    .. warn:: This function operates by side-effect.  The plugin_info dictionay is modified
+    .. warning:: This function operates by side-effect.  The plugin_info dictionay is modified
         directly.
     """
     for plugin_type, plugin_map in plugin_info.items():

antsibull/cli/antsibull_docs.py

@@ -23,6 +23,7 @@
 from ..args import (  # noqa: E402
     InvalidArgumentError, get_toplevel_parser, normalize_toplevel_options
 )
+from ..compat import BooleanOptionalAction  # noqa: E402
 from ..config import load_config  # noqa: E402
 from ..constants import DOCUMENTABLE_PLUGINS  # noqa: E402
 from ..filesystem import UnableToCheck, writable_via_acls  # noqa: E402
@@ -143,6 +144,16 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
     flog = mlog.fields(func='parse_args')
     flog.fields(program_name=program_name, raw_args=args).info('Enter')
 
+    # Overview of parsers:
+    # * docs_parser is an abstract parser.  Contains options that all of the antisbull-docs
+    #   subcommands use.
+    # * cache_parser is a mixin for subcommands which operate on the ansible-core sources and
+    #   therefore they can use a preinstalled version of the code instead of downloading it
+    #   themselves.
+    # * whole_site_parser is a mixin for subcommands which can choose to build a structure
+    #   for integration into a comprehensive website.
+    # * devel, stable, current, collection, file: These parsers contain all of the options for those
+    #   respective subcommands.
     docs_parser = argparse.ArgumentParser(add_help=False)
     docs_parser.add_argument('--dest-dir', default='.',
                              help='Directory to write the output to')
@@ -160,28 +171,58 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
                               ' of downloading fresh versions provided that they meet the criteria'
                               ' (Latest version of the collections known to galaxy).')
 
+    whole_site_parser = argparse.ArgumentParser(add_help=False)
+    whole_site_parser.add_argument('--breadcrumbs', '--gretel',
+                                   dest='breadcrumbs', action=BooleanOptionalAction,
+                                   default=argparse.SUPPRESS,
+                                   help='Determines whether to add breadcrumbs to plugin docs via'
+                                   ' hidden sphinx toctrees. This can take up a significant'
+                                   ' amount of memory if there are a large number of plugins so'
+                                   ' you can disable this if necessary. (default: True)')
+    whole_site_parser.add_argument('--indexes',
+                                   dest='indexes', action=BooleanOptionalAction,
+                                   default=argparse.SUPPRESS,
+                                   help='Determines whether to create the collection index and'
+                                   ' plugin indexes. They may not be needed if you have'
+                                   ' a different structure for your website. (default: True)')
+    # --skip-indexes is for backwards compat.  We want all negations to be --no-* in the future.
+    whole_site_parser.add_argument('--skip-indexes',
+                                   dest='indexes', action='store_false',
+                                   default=argparse.SUPPRESS,
+                                   help='Do not create the collection index and plugin indexes.'
+                                   ' This option is deprecated in favor of --no-indexes')
+
     parser = get_toplevel_parser(prog=program_name,
                                  description='Script to manage generated documentation for'
                                  ' ansible')
     subparsers = parser.add_subparsers(title='Subcommands', dest='command',
-                                       help='for help use  SUBCOMMANDS -h')
+                                       help='for help use: `SUBCOMMANDS -h`')
     subparsers.required = True
 
+    #
     # Document the next version of ansible
-    devel_parser = subparsers.add_parser('devel', parents=[docs_parser, cache_parser],
+    #
+    devel_parser = subparsers.add_parser('devel',
+                                         parents=[docs_parser, cache_parser, whole_site_parser],
                                          description='Generate documentation for the next major'
                                          ' release of Ansible')
     devel_parser.add_argument('--pieces-file', default=DEFAULT_PIECES_FILE,
                               help='File containing a list of collections to include')
 
+    #
+    # Document a released version of ansible
+    #
     stable_parser = subparsers.add_parser('stable',
-                                          parents=[docs_parser, cache_parser],
+                                          parents=[docs_parser, cache_parser, whole_site_parser],
                                           description='Generate documentation for a current'
                                           ' version of ansible')
     stable_parser.add_argument('--deps-file', required=True,
                                help='File which contains the list of collections and'
                                ' versions which were included in this version of Ansible')
 
+    #
+    # Document the currently installed version of ansible
+    #
     current_parser = subparsers.add_parser('current',
                                            parents=[docs_parser],
                                            description='Generate documentation for the current'
@@ -192,10 +233,13 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
                                 ' specified, all collections in the currently configured ansible'
                                 ' search paths will be used')
 
+    #
+    # Document one or more specified collections
+    #
     collection_parser = subparsers.add_parser('collection',
-                                              parents=[docs_parser],
-                                              description='Generate documentation for a single'
-                                              ' collection')
+                                              parents=[docs_parser, whole_site_parser],
+                                              description='Generate documentation for specified'
+                                              ' collections')
     collection_parser.add_argument('--collection-version', default='@latest',
                                    help='The version of the collection to document.  The special'
                                    ' version, "@latest" can be used to download and document the'
@@ -205,19 +249,19 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
                                    ' these collections have been installed with the current'
                                    ' version of ansible. Specified --collection-version will be'
                                    ' ignored.')
-    collection_parser.add_argument('--skip-indexes', action='store_true',
-                                   help='Do not create the collection index and plugin indexes.'
-                                   ' Automatically assumed when --squash-hierarchy is specified.')
     collection_parser.add_argument('--squash-hierarchy', action='store_true',
                                    help='Do not use the full hierarchy collections/namespace/name/'
                                    ' in the destination directory. Only valid if there is only'
-                                   ' one collection specified.')
+                                   ' one collection specified.  Implies --no-indexes.')
     collection_parser.add_argument(nargs='+', dest='collections',
                                    help='One or more collections to document.  If the names are'
                                    ' directories on disk, they will be parsed as expanded'
                                    ' collections. Otherwise, if they could be collection'
                                    ' names, they will be downloaded from galaxy.')
 
+    #
+    # Document a specifically named plugin
+    #
     file_parser = subparsers.add_parser('plugin',
                                         parents=[docs_parser],
                                         description='Generate documentation for a single plugin')
@@ -235,6 +279,14 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
         flog.warning('The CLI parameter, `--ansible-base-cache` has been renamed to'
                      ' `--ansible-base-source.  Please use that instead')
 
+    if '--skip-indexes' in args:
+        flog.warning('The CLI parameter, `--skip-indexes` has been renamed to'
+                     ' `--no-indexes`.  Please use that instead')
+        if '--indexes' or '--no-indexes' in args:
+            raise InvalidArgumentError('You cannot use `--indexes`/`--no-indexes` with'
+                                       ' `--skip-indexes`. Please remove `--skip-indexes`'
+                                       ' and try again.')
+
     args: argparse.Namespace = parser.parse_args(args)
     flog.fields(args=args).debug('Arguments parsed')
 
@@ -270,6 +322,7 @@ def run(args: List[str]) -> int:
     try:
         args: argparse.Namespace = parse_args(program_name, args[1:])
     except InvalidArgumentError as e:
+        flog.error(e)
         print(e)
         return 2
     flog.fields(args=args).info('Arguments parsed')

antsibull/cli/doc_commands/collection.py

@@ -12,18 +12,20 @@
 mlog = log.fields(mod=__name__)
 
 
-def generate_current_docs(skip_indexes: bool, squash_hierarchy: bool) -> int:
+def generate_current_docs(indexes: bool, squash_hierarchy: bool) -> int:
     flog = mlog.fields(func='generate_current_docs')
     flog.debug('Begin processing docs')
 
     app_ctx = app_context.app_ctx.get()
+    lib_ctx = app_context.lib_ctx.get()
 
     venv = FakeVenvRunner()
 
     generate_docs_for_all_collections(
         venv, None, app_ctx.extra['dest_dir'], app_ctx.extra['collections'],
-        create_indexes=not skip_indexes and not squash_hierarchy,
-        squash_hierarchy=squash_hierarchy)
+        create_indexes=indexes and not squash_hierarchy,
+        squash_hierarchy=squash_hierarchy,
+        breadcrumbs=lib_ctx.breadcrumbs)
 
     return 0
 
@@ -41,11 +43,11 @@ def generate_docs() -> int:
     """
     app_ctx = app_context.app_ctx.get()
 
-    skip_indexes: bool = app_ctx.extra['skip_indexes']
+    indexes: bool = app_ctx.indexes
     squash_hierarchy: bool = app_ctx.extra['squash_hierarchy']
 
     if app_ctx.extra['use_current']:
-        return generate_current_docs(skip_indexes, squash_hierarchy)
+        return generate_current_docs(indexes, squash_hierarchy)
 
     raise NotImplementedError('Priority to implement subcommands is stable, devel, plugin, and'
                               ' then collection commands. Only --use-current is implemented'

antsibull/cli/doc_commands/current.py

@@ -26,10 +26,12 @@ def generate_docs() -> int:
     flog.debug('Begin processing docs')
 
     app_ctx = app_context.app_ctx.get()
+    lib_ctx = app_context.lib_ctx.get()
 
     venv = FakeVenvRunner()
 
     generate_docs_for_all_collections(
-        venv, app_ctx.extra['collection_dir'], app_ctx.extra['dest_dir'])
+        venv, app_ctx.extra['collection_dir'], app_ctx.extra['dest_dir'],
+        breadcrumbs=lib_ctx.breadcrumbs)
 
     return 0

antsibull/cli/doc_commands/devel.py

@@ -136,6 +136,7 @@ def generate_docs() -> int:
         venv.install_package(ansible_base_path)
         flog.fields(venv=venv).notice('Finished installing ansible-core')
 
-        generate_docs_for_all_collections(venv, collection_dir, app_ctx.extra['dest_dir'])
+        generate_docs_for_all_collections(venv, collection_dir, app_ctx.extra['dest_dir'],
+                                          breadcrumbs=app_ctx.breadcrumbs)
 
     return 0