Generate galaxy.yml based on single source of truth

.gitignore

@@ -34,6 +34,7 @@ docs/docsite/*.html
 docs/docsite/htmlout
 docs/docsite/rst/cli/ansible-*.rst
 docs/docsite/rst/cli/ansible.rst
+docs/docsite/rst/dev_guide/collections_galaxy_meta.rst
 docs/docsite/rst/dev_guide/testing/sanity/index.rst.new
 docs/docsite/rst/modules/*.rst
 docs/docsite/rst/playbooks_directives.rst

docs/docsite/Makefile

@@ -5,6 +5,7 @@ TESTING_FORMATTER=../bin/testing_formatter.sh
 KEYWORD_DUMPER=../../hacking/build-ansible.py document-keywords
 CONFIG_DUMPER=../../hacking/build-ansible.py document-config
 GENERATE_CLI=../../hacking/build-ansible.py generate-man
+COLLECTION_DUMPER=../../hacking/build-ansible.py collection-meta
 ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1)
 CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}')
 else
@@ -37,7 +38,7 @@ all: docs
 
 docs: htmldocs
 
-generate_rst: config cli keywords modules plugins testing
+generate_rst: collections_meta config cli keywords modules plugins testing
 
 htmldocs: generate_rst
 	CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx html
@@ -75,9 +76,13 @@ clean:
 	rm -f rst/plugins/*/*.rst
 	rm -f rst/reference_appendices/config.rst
 	rm -f rst/reference_appendices/playbooks_keywords.rst
+	rm -f rst/dev_guide/collections_galaxy_meta.rst
 
 .PHONY: docs clean
 
+collections_meta: ../templates/collections_galaxy_meta.rst.j2
+	PYTHONPATH=../../lib $(COLLECTION_DUMPER) --template-file=../templates/collections_galaxy_meta.rst.j2 --output-dir=rst/dev_guide/ ../../lib/ansible/galaxy/data/collections_galaxy_meta.yml
+
 # TODO: make generate_man output dir cli option
 cli:
 	mkdir -p rst/cli

docs/docsite/rst/collections_tech_preview.rst → docs/docsite/rst/dev_guide/collections_tech_preview.rst

@@ -55,50 +55,9 @@ and other tools need in order to package, build and publish the collection.::
 galaxy.yml
 ----------
 
-This file contains the information about a collection that is necessary for Ansible tools to operate.
-``galaxy.yml`` has the following fields (subject to changes and expansion):
+A collection must have a ``galaxy.yml`` file that contains the necessary information to build a collection artifact.
+See :ref:`collections_galaxy_meta` for details on how this file is structured.
 
-.. code-block:: yaml
-
-    namespace: "namespace_name"
-    name: "collection_name"
-    version: "1.0.12"
-    authors:
-        - "Author1"
-        - "Author2 (https://author2.example.com)"
-        - "Author3 <author3@example.com>"
-    dependencies:
-        "other_namespace.collection1": ">=1.0.0"
-        "other_namespace.collection2": ">=2.0.0,<3.0.0"
-        "anderson55.my_collection": "*"    # note: "*" selects the highest version available
-    license:
-        - "MIT"
-    tags:
-        - demo
-        - collection
-    repository: "https://www.github.com/my_org/my_collection"
-
-
-Required Fields:
-    - ``namespace``: the namespace that the collection lives under. It must be a valid Python identifier,
-        and may only contain alphanumeric characters and underscores. Additionally
-        the ``namespace`` cannot start with underscores or numbers and cannot contain consecutive
-        underscores.
-    - ``name``: the collection's name. Has the same character restrictions as ``namespace``.
-    - ``version``: the collection's version. To upload to Galaxy, it must be compatible with semantic versioning.
-
-
-Optional Fields:
-    - ``dependencies``: A dictionary where keys are collections, and values are version
-      range `specifiers <https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification>`_.
-      It is good practice to depend on a version range to minimize conflicts, and pin to a
-      a major version to protect against breaking changes. For example: ``"user1.collection1": ">=1.2.2,<2.0.0"``
-      This field allows other collections as dependencies, not traditional roles.
-    - ``description``: A short summary description of the collection.
-    - ``license``: Either a single license or a list of licenses for content inside of a collection.
-      Galaxy currently only accepts `SPDX <https://spdx.org/licenses/>`_ licenses.
-    - ``tags``: a list of tags. These have the same character requirements as ``namespace`` and ``name``.
-    - ``repository``: URL of originating SCM repository.
 
 docs directory
 ---------------

docs/docsite/rst/dev_guide/index.rst

@@ -81,4 +81,6 @@ If you prefer to read the entire guide, here's a list of the pages in order.
    developing_api
    developing_rebasing
    developing_module_utilities
+   collections_tech_preview
+   collections_galaxy_meta
    overview_architecture

docs/templates/collections_galaxy_meta.rst.j2

@@ -0,0 +1,74 @@
+.. _collections_galaxy_meta:
+
+************************************
+Collection Galaxy Metadata Structure
+************************************
+
+.. important::
+    This feature is available in Ansible 2.8 as a *Technology Preview* and therefore is not fully supported. It should only be used for testing  and should not be deployed in a production environment.
+    Future Galaxy or Ansible releases may introduce breaking changes.
+
+A key component of an Ansible collection is the ``galaxy.yml`` file placed in the root directory of a collection. This
+file contains the metadata of the collection that is used to generate a collection artifact.
+
+Structure
+=========
+
+The ``galaxy.yml`` file must contain the following keys in valid YAML:
+
+.. raw:: html
+
+    <table  border=0 cellpadding=0 class="documentation-table">
+        {# Header of the documentation -#}
+        <tr>
+            <th>Key</th>
+            <th width="100%">Comments</th>
+        </tr>
+        {% for entry in options %}
+            <tr>
+                {# key name with required or type label #}
+                <td>
+                    <b>@{ entry.key }@</b>
+                    <div style="font-size: small">
+                        <span style="color: purple">@{ entry.type | documented_type }@</span>
+                        {% if entry.get('required', False) %} / <span style="color: red">required</span>{% endif %}
+                    </div>
+                </td>
+                {# Comments #}
+                <td>
+                    {% if entry.description is string %}
+                        <div>@{ entry.description | replace('\n', '\n    ') | html_ify }@</div>
+                    {% else %}
+                        {% for desc in entry.description %}
+                            <div>@{ desc | replace('\n', '\n    ') | html_ify }@</div>
+                        {% endfor %}
+                    {% endif %}
+                </td>
+            </tr>
+        {% endfor %}
+    </table>
+    <br/>
+
+Examples
+========
+
+.. code-block:: yaml
+
+    namespace: "namespace_name"
+    name: "collection_name"
+    version: "1.0.12"
+    readme: "README.md"
+    authors:
+        - "Author1"
+        - "Author2 (https://author2.example.com)"
+        - "Author3 <author3@example.com>"
+    dependencies:
+        "other_namespace.collection1": ">=1.0.0"
+        "other_namespace.collection2": ">=2.0.0,<3.0.0"
+        "anderson55.my_collection": "*"    # note: "*" selects the highest version available
+    license:
+        - "MIT"
+    tags:
+        - demo
+        - collection
+    repository: "https://www.github.com/my_org/my_collection"

hacking/build_library/build_ansible/command_plugins/collection_meta.py

@@ -0,0 +1,68 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import os.path
+import pathlib
+
+import yaml
+from jinja2 import Environment, FileSystemLoader
+from ansible.module_utils._text import to_bytes
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+from ..jinja2.filters import documented_type, html_ify  # pylint: disable=relative-beyond-top-level
+
+
+DEFAULT_TEMPLATE_FILE = 'collections_galaxy_meta.rst.j2'
+DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates'
+
+
+class DocumentCollectionMeta(Command):
+    name = 'collection-meta'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name, description='Generate collection galaxy.yml documentation from shared metadata')
+        parser.add_argument("-t", "--template-file", action="store", dest="template_file",
+                            default=DEFAULT_TEMPLATE_FILE,
+                            help="Jinja2 template to use for the config")
+        parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
+                            default=DEFAULT_TEMPLATE_DIR,
+                            help="directory containing Jinja2 templates")
+        parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/',
+                            help="Output directory for rst files")
+        parser.add_argument("collection_defs", metavar="COLLECTION-OPTION-DEFINITIONS.yml", type=str,
+                            help="Source for collection metadata option docs")
+
+    @staticmethod
+    def main(args):
+        output_dir = os.path.abspath(args.output_dir)
+        template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file))
+        template_file = os.path.basename(template_file_full_path)
+        template_dir = os.path.dirname(template_file_full_path)
+
+        with open(args.collection_defs) as f:
+            options = yaml.safe_load(f)
+
+        env = Environment(loader=FileSystemLoader(template_dir),
+                          variable_start_string="@{",
+                          variable_end_string="}@",
+                          trim_blocks=True)
+        env.filters['documented_type'] = documented_type
+        env.filters['html_ify'] = html_ify
+
+        template = env.get_template(template_file)
+        output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
+        temp_vars = {'options': options}
+
+        data = to_bytes(template.render(temp_vars))
+        update_file_if_different(output_name, data)
+
+        return 0

hacking/build_library/build_ansible/command_plugins/plugin_formatter.py

@@ -11,7 +11,6 @@
 import datetime
 import glob
 import json
-import optparse
 import os
 import re
 import sys
@@ -34,10 +33,9 @@ def html_escape(text, quote=True):
 import jinja2
 import yaml
 from jinja2 import Environment, FileSystemLoader
-from jinja2.runtime import Undefined
 
 from ansible.errors import AnsibleError
-from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils._text import to_bytes
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import iteritems, string_types
@@ -48,6 +46,7 @@ def html_escape(text, quote=True):
 # Pylint doesn't understand Python3 namespace modules.
 from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
 from ..commands import Command  # pylint: disable=relative-beyond-top-level
+from ..jinja2.filters import do_max, documented_type, html_ify, rst_fmt, rst_ify, rst_xline  # pylint: disable=relative-beyond-top-level
 
 
 #####################################################################################
@@ -67,14 +66,6 @@ def html_escape(text, quote=True):
     os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
 ))
 
-_ITALIC = re.compile(r"I\(([^)]+)\)")
-_BOLD = re.compile(r"B\(([^)]+)\)")
-_MODULE = re.compile(r"M\(([^)]+)\)")
-_URL = re.compile(r"U\(([^)]+)\)")
-_LINK = re.compile(r"L\(([^)]+),([^)]+)\)")
-_CONST = re.compile(r"C\(([^)]+)\)")
-_RULER = re.compile(r"HORIZONTALLINE")
-
 DEPRECATED = b" (D)"
 
 pp = PrettyPrinter()
@@ -98,74 +89,6 @@ def from_kludge_ns(key):
     return NS_MAP[key]
 
 
-# The max filter was added in Jinja2-2.10.  Until we can require that version, use this
-def do_max(seq):
-    return max(seq)
-
-
-def rst_ify(text):
-    ''' convert symbols like I(this is in italics) to valid restructured text '''
-
-    try:
-        t = _ITALIC.sub(r"*\1*", text)
-        t = _BOLD.sub(r"**\1**", t)
-        t = _MODULE.sub(r":ref:`\1 <\1_module>`", t)
-        t = _LINK.sub(r"`\1 <\2>`_", t)
-        t = _URL.sub(r"\1", t)
-        t = _CONST.sub(r"``\1``", t)
-        t = _RULER.sub(r"------------", t)
-    except Exception as e:
-        raise AnsibleError("Could not process (%s) : %s" % (text, e))
-
-    return t
-
-
-def html_ify(text):
-    ''' convert symbols like I(this is in italics) to valid HTML '''
-
-    if not isinstance(text, string_types):
-        text = to_text(text)
-
-    t = html_escape(text)
-    t = _ITALIC.sub(r"<em>\1</em>", t)
-    t = _BOLD.sub(r"<b>\1</b>", t)
-    t = _MODULE.sub(r"<span class='module'>\1</span>", t)
-    t = _URL.sub(r"<a href='\1'>\1</a>", t)
-    t = _LINK.sub(r"<a href='\2'>\1</a>", t)
-    t = _CONST.sub(r"<code>\1</code>", t)
-    t = _RULER.sub(r"<hr/>", t)
-
-    return t.strip()
-
-
-def rst_fmt(text, fmt):
-    ''' helper for Jinja2 to do format strings '''
-
-    return fmt % (text)
-
-
-def rst_xline(width, char="="):
-    ''' return a restructured text line of a given length '''
-
-    return char * width
-
-
-def documented_type(text):
-    ''' Convert any python type to a type for documentation '''
-
-    if isinstance(text, Undefined):
-        return '-'
-    if text == 'str':
-        return 'string'
-    if text == 'bool':
-        return 'boolean'
-    if text == 'int':
-        return 'integer'
-    if text == 'dict':
-        return 'dictionary'
-    return text
-
-
 test_list = partial(is_sequence, include_strings=False)