[TSC]: Build with readthedocs

build/direct-copy/direct-copy.py

@@ -0,0 +1,34 @@
+####
+# A sphinx extension to copy files directly from the source directory to the target
+# directory. 
+# Usage: 
+#  1) Enable `direct-copy` extension in your conf.py
+#  2) Add a list of directories to be copyed in the `direct_copy_directories` parameter.
+####
+
+from docutils import nodes
+from docutils.parsers.rst import Directive
+from sphinx.locale import _
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util import logging
+
+import shutil
+
+logger = logging.getLogger(__name__)
+
+def direct_copy(app, exception):
+    logger.info("direct_copy source directory is " + app.srcdir + ", target directory is " + app.outdir)
+    for dir in app.config.direct_copy_directories:
+        logger.info('Direct copying '+ dir)
+        shutil.copytree(src=app.srcdir + dir, dst=app.outdir + dir, symlinks=True)
+
+def setup(app):
+    app.add_config_value('direct_copy_directories', False, 'html')
+
+    app.connect('build-finished', direct_copy)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

build/readme-to-index/readme-to-index.py

@@ -0,0 +1,39 @@
+###
+# A sphinx extension to copy generated README.html files to index.html.
+###
+
+from docutils import nodes
+from docutils.parsers.rst import Directive
+from sphinx.locale import _
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util import logging
+
+import shutil
+import os
+logger = logging.getLogger(__name__)
+
+
+def readme_to_index(app, exception):
+    logger.info("readme_to_index target directory is " + app.outdir)
+    for dirpath, dirnames, filenames in os.walk(app.outdir):
+        for name in filenames:
+            if name == "README.html":
+                logger.info(" -" + os.path.join(dirpath, name))
+                target = dirpath + "/index.html"
+                source = dirpath + "/README.html"
+                if not os.path.isfile(target):
+                    logger.info("Copying " + source + " to " + target)
+                    shutil.copy(source, target, follow_symlinks=True)
+                else:
+                    logger.info("  -" + target + " already exist. NOT writing it over.")
+
+
+def setup(app):
+
+    app.connect('build-finished', readme_to_index)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

build/relative-link-corrector/relative-link-corrector.py

@@ -0,0 +1,89 @@
+###
+# A sphinx extension to support the relative linking of html files.
+# Inspired by and partly copyed from 
+# https://github.com/firegurafiku/sphinxcontrib-divparams/
+###
+
+#from docutils import nodes
+#from docutils.parsers.rst import Directive
+#from sphinx.locale import _
+#from sphinx.util.docutils import SphinxDirective
+from sphinx.util import logging
+from sphinx.environment import NoUri
+
+import bs4
+import shutil
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+def transform_html(soup):
+    links = soup.find_all("a")
+    for link in links:
+        #logger.info("l: " + str(link))
+        if link.has_attr("href"):
+            logger.debug("l: " + str(link['href']))
+            if "://" not in link['href']:
+                res = re.search("(\.md\#|\.md$)", link['href'])
+                if res:
+                    corect_link = link['href'].replace(".md", ".html")
+                    logger.debug("  c: correct link is " + corect_link)
+                    link['href'] = corect_link
+                else: 
+                    logger.debug("  c: no match of .md")
+            else: 
+                logger.debug("  c: this is an absolulte link")
+
+        logger.debug("l: done")
+
+def relative_link_corrector(app, exception):
+    logger.info("relative-link-corrector is correcting relative links")
+
+    # Don't risk doing anything if Sphinx failed in building HTML.
+    if exception is not None:
+        return
+
+    # Find all files eligible for DOM transform.
+    # See also: http://stackoverflow.com/a/33640970/1447225
+    target_files = []
+    for doc in app.env.found_docs:
+        target_filename = "#"
+        try:
+            target_filename = app.builder.get_target_uri(doc)
+        except NoUri:
+            continue
+        if '#' in target_filename:
+            logger.info("Skipping")
+            continue
+
+#        logger.info("tfn1: " + target_filename)
+        target_filename = os.path.join(app.outdir, target_filename)
+#        logger.info("tfn2: " + target_filename)
+        target_filename = os.path.abspath(target_filename)
+#        logger.info("tfn3: " + target_filename)
+        target_files.append(target_filename)
+
+    for fn in target_files:
+        try:
+            with open(fn, mode="rb") as f:
+                soup = bs4.BeautifulSoup(f.read(), "html.parser")
+
+            transform_html(soup)
+            html = soup.prettify(encoding=app.config.html_output_encoding)
+
+            with open(fn, mode='wb') as f:
+                f.write(html)
+
+        except Exception as exc:
+            app.warning("Exception raised during HTML tweaking: " + str(exc))
+
+def setup(app):
+    app.connect('build-finished', relative_link_corrector)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

doc/README.md

@@ -1,14 +1,14 @@
 # Cloud iNfrastructure Telco Task Force (CNTT)
 
 ## Table of Contents
-* [Governance & Community Guidelines](gov)
+* [Governance & Community Guidelines](gov/README.md)
   * [Terms of Reference](../GSMA_CNTT_Terms_of_Reference.pdf)
   * [Code of Conduct](../CODE_OF_CONDUCT.md)
   * [The License (Creative Commons Attribution 4.0 International)](https://creativecommons.org/licenses/by/4.0/legalcode)
-* [Technical Specifications](common)
+* [Technical Specifications](common/README.md)
   * [Introduction](common/chapter00.md)
-  * [Reference Model](ref_model)
-  * [Reference Architectures](ref_arch)
-  * [Reference Implementations](ref_impl)
-    * [Vendor Implementations](ven_impl)
-  * [Reference Conformances](ref_cert)
+  * [Reference Model](ref_model/README.md)
+  * [Reference Architectures](ref_arch/README.md)
+  * [Reference Implementations](ref_impl/README.md)
+    * [Vendor Implementations](ven_impl/README.md)
+  * [Reference Conformances](ref_cert/README.md)

doc/conf.py

@@ -0,0 +1,35 @@
+import os
+import sys
+import sphinx_rtd_theme
+
+from recommonmark.parser import CommonMarkParser
+source_parsers = {
+'.md': CommonMarkParser,
+}
+source_suffix = ['.rst', '.md']
+master_doc = 'index'
+project = u'CNTT-CNTT'
+
+sys.path.append(os.path.abspath("../build/direct-copy"))
+sys.path.append(os.path.abspath("../build/readme-to-index"))
+sys.path.append(os.path.abspath("../build/relative-link-corrector"))
+
+extensions = ['direct-copy', 
+              'readme-to-index', 
+              'relative-link-corrector',
+              'sphinx_markdown_tables', 
+              'sphinx_rtd_theme']
+
+direct_copy_directories = ['/gov/figures', 
+                           '/ref_model/figures', 
+                           '/ref_arch/figures', 
+                           '/ref_arch/kubernetes/figures', 
+                           '/ref_arch/openstack/figures', 
+                           '/ref_cert/RC1/figures', 
+                           '/ref_cert/RC2/figures', 
+                           '/ref_impl/cntt-ri/figures', 
+                           '/ref_impl/cntt-ri2/figures',
+                           '/ven_impl/figures', 
+                           '/common/figures'] 
+
+html_theme = "sphinx_rtd_theme"

doc/index.rst

@@ -0,0 +1,15 @@
+CNTT
+----
+
+Documents
+=========
+.. toctree::
+    :maxdepth: 16
+
+    /common/README.md
+    /gov/README.md
+    /ref_model/README.md
+    /ref_arch/README.md
+    /ref_impl/README.md
+    /ref_cert/README.md
+    /ven_impl/README.md

doc/ref_arch/openstack/chapters/chapter02.md

@@ -481,9 +481,8 @@ The Platform is assumed to provide configurable alerting and notification capabi
 | sec.std.016 | Standards | The Cloud Operator **should** conform to the ISO/IEC 27035 standard for incidence management; ISO/IEC 27035 - ISO/IEC 27035 is the international Standard for incident management | |
 | sec.std.017 | Standards | The Cloud Operator **should** conform to the ISO/IEC 27031 standard for business continuity; ISO/IEC 27031 - ISO/IEC 27031 is the international Standard for ICT readiness for business continuity | |
 
-
-
 <p align="center"><b>Table 2-16:</b> Security Recommendations</p>
+
 <!--
 **Backlog of Req**
 
@@ -549,4 +548,6 @@ The Platform is assumed to provide configurable alerting and notification capabi
 
 1. Logging, Monitoring, Alerting of the Cloud should ensure any failures in the control plane are either self-healed or alerted on and ideally some sort of centralised log file analysis should be possible without needing to trawl local log files.    Logging in the tenant space is left to the application.
 1. Backup of the control plane configuration (keystone DB, other DB, policy.json’s) to a remote object store.
+
 -->
+

doc/ref_arch/openstack/chapters/chapter04.md

@@ -389,11 +389,9 @@ As we have seen a flavor series is supported by configuring hosts in accordance
 
 #### 4.2.3.1 Physical Network Topology
 
-
-
 #### 4.2.3.2 High Level Logical Network Layout
 
-<p align="center"><img src="../figures/RA1-Ch04-Indicative-OpenStack-Network.png" alt="Indicative OpenStack Network Layout"></br>Figure 4-5: Indicative OpenStack Network Layout.</p>
+<p align="center"><img src="../figures/RA1-Ch04-Indicative-OpenStack-Network.png" alt="Indicative OpenStack Network Layout"><b>Figure 4-5: Indicative OpenStack Network Layout.</b></p>
 
 | Network | Description | Characteristics |
 |----------|---------|--------------|
@@ -467,7 +465,7 @@ The Ceph storage cluster is deployed on bare metal hardware. The minimal configu
 
 Ceph monitors maintain a master copy of the maps of the cluster state required by Ceph daemons to coordinate with each other. Ceph OSD handle the data storage (read/write data on the physical disks), data replication, recovery, rebalancing, and provides some monitoring information to Ceph Monitors. The RadosGW provides Object Storage RESTful gateway with a Swift-compatible API for Object Storage.
 
-<p align="center"><img src="../figures/RA1-Ch04-Ceph.png" alt="Ceph Storage System"></br>Figure 4-6: Ceph Storage System.</p>
+<p align="center"><img src="../figures/RA1-Ch04-Ceph.png" alt="Ceph Storage System"><b>Figure 4-6: Ceph Storage System.</b></p>
 
 **BIOS Requirement for Ceph servers**
 
@@ -534,9 +532,9 @@ Neutron is the networking service, Neutron depends on Keystone and has services
 | Networking Service component | Description | Required or Optional Service | Placement |
 |-----|-----|----|----|
 | neutron server (neutron-server and neutron-\*-plugin) | Manages user requests and exposes the Neutron APIs | Required | Controller node |
-| DHCP agent (neutron-dhcp-agent) | Provides DHCP services to tenant networks and is responsible for maintaining DHCP configuration. For High availability, multiple DHCP agents can be assigned. | Optional depending upon plug-in | Network node </br> (Controller node if no network node present) |
-| L3 agent (neutron-l3-agent) | Provides L3/NAT forwarding for external network access of VMs on tenant networks and supports services such as Firewall-as-a-service (FWaaS) and Load Balancer-as-a-service (LBaaS) | Optional depending upon plug-in | Network node </br>(Controller node if no network node present)</br> NB in DVR based OpenStack Networking, also in all Compute nodes. |
-| neutron metadata agent (neutron-metadata-agent) | The metadata service provides a way for instances to retrieve instance-specific data. The networking service, neutron, is responsible for intercepting these requests and adding HTTP headers which uniquely identify the source of the request before forwarding it to the metadata API server. These functions are performed by the neutron metadata agent. | Optional | Network node </br> (Controller node if no network node present) |
+| DHCP agent (neutron-dhcp-agent) | Provides DHCP services to tenant networks and is responsible for maintaining DHCP configuration. For High availability, multiple DHCP agents can be assigned. | Optional depending upon plug-in | Network node <br> (Controller node if no network node present) |
+| L3 agent (neutron-l3-agent) | Provides L3/NAT forwarding for external network access of VMs on tenant networks and supports services such as Firewall-as-a-service (FWaaS) and Load Balancer-as-a-service (LBaaS) | Optional depending upon plug-in | Network node <br>(Controller node if no network node present)<br> NB in DVR based OpenStack Networking, also in all Compute nodes. |
+| neutron metadata agent (neutron-metadata-agent) | The metadata service provides a way for instances to retrieve instance-specific data. The networking service, neutron, is responsible for intercepting these requests and adding HTTP headers which uniquely identify the source of the request before forwarding it to the metadata API server. These functions are performed by the neutron metadata agent. | Optional | Network node <br> (Controller node if no network node present) |
 | neutron plugin agent (neutron-\*-agent) | Runs on each compute node to control and manage the local virtual network driver (such as the Open vSwitch or Linux Bridge) configuration and local networking configuration for VMs hosted on that node. | Required | Every Compute Node |
 <p align="center"><b>Table 4-2: Neutron Services Placement</b></p>
 
@@ -627,7 +625,7 @@ Containers are lightweight compared to Virtual Machines and leads to efficient r
 
 In Chapter 3, [Figure 3.2](../figures/RA1-Ch03-OpenStack-Services-Topology.png) shows a high level Virtualised OpenStack services topology. The containerized OpenStack services topology version is shown in Figure 4-7.
 
-<p align="center"><img src="../figures/RA1-Ch04-Containerised-OpenStack-Services-Stack.png" alt="Containerised OpenStack Services Topology"></br>Figure 4-7: Containerised OpenStack Services Topology.</p>
+<p align="center"><img src="../figures/RA1-Ch04-Containerised-OpenStack-Services-Stack.png" alt="Containerised OpenStack Services Topology"><br>Figure 4-7: Containerised OpenStack Services Topology.</p>
 
 
 <a name="4.4"></a>

doc/ref_impl/cntt-ri/chapters/chapter03.md

@@ -334,22 +334,19 @@ Consumable Infrastructure Resources and Services
 
 ## 3.8 Interface and API for Reference Implementation 1
 
-The following table lists the interface for RI1. 
-| OpenStack Service     | Link for API list                                    | **API Version** | **Minimal API Microversion** |
-|-----------------------|------------------------------------------------------|-----------------|------------------------------|
-| Identity: Keystone    | https://docs.openstack.org/api-ref/identity/v3/      | 3               | 3.8                          |
-| Compute: Nova         | https://docs.openstack.org/api-ref/compute/          | v2.1            | 2.53                         |
-| Networking: Neutron   | https://docs.openstack.org/api-ref/network/v2/       | v2.0            |                              |
-| Image: Glance         | https://docs.openstack.org/api-ref/image/v2/         | v2              | 2.5                          |
-| Block Storage: Cinder | https://docs.openstack.org/api-ref/block-storage/v3/ | v3              | 3.43                         |
-| Object Storage: Swift | https://docs.openstack.org/api-ref/object-store/     | v1              |                              |
-| Placement             | https://docs.openstack.org/api-ref/placement/        | v1              | 1.10                         |
-| Orchestration: Heat   | https://docs.openstack.org/api-ref/orchestration/v1/ | v1              |                              |
-| Acceleration: Cyborg  | https://docs.openstack.org/api-ref/accelerator/v2/ | v2    |   |
-|K8S API  |https://kubernetes.io/docs/concepts/overview/kubernetes-api/|  |  |
-|KVM APIs  |https://www.kernel.org/doc/Documentation/virtual/kvm/api.txt|  |  |
-| Libvirt APIs |https://libvirt.org/html/index.html|  |  |
-
-
-
+The following table lists the interface for RI1.
 
+| OpenStack Service     | Link for API list                                    | **API Version** | **Minimal API Microversion** |
+|-----------------------|------------------------------------------------------|------|------|
+| Identity: Keystone    | https://docs.openstack.org/api-ref/identity/v3/      | 3    | 3.8  |
+| Compute: Nova         | https://docs.openstack.org/api-ref/compute/          | v2.1 | 2.53 |
+| Networking: Neutron   | https://docs.openstack.org/api-ref/network/v2/       | v2.0 |      |
+| Image: Glance         | https://docs.openstack.org/api-ref/image/v2/         | v2   | 2.5  |
+| Block Storage: Cinder | https://docs.openstack.org/api-ref/block-storage/v3/ | v3   | 3.43 |
+| Object Storage: Swift | https://docs.openstack.org/api-ref/object-store/     | v1   |      |
+| Placement             | https://docs.openstack.org/api-ref/placement/        | v1   | 1.10 |
+| Orchestration: Heat   | https://docs.openstack.org/api-ref/orchestration/v1/ | v1   |      |
+| Acceleration: Cyborg  | https://docs.openstack.org/api-ref/accelerator/v2/   | v2   |      |
+| K8S API               | https://kubernetes.io/docs/concepts/overview/kubernetes-api/ |  |  |
+| KVM APIs              | https://www.kernel.org/doc/Documentation/virtual/kvm/api.txt |  |  |
+| Libvirt APIs          | https://libvirt.org/html/index.html                  |      |      |

requirements.txt

@@ -5,5 +5,9 @@ mock==1.0.1
 pillow==5.4.1
 alabaster>=0.7,<0.8,!=0.7.5
 commonmark==0.8.1
-recommonmark==0.5.0
-mkdocs<1.1
+recommonmark==0.6.0
+sphinx<2
+readthedocs-sphinx-ext<1.1
+sphinx-markdown-tables==0.0.15
+beautifulsoup4==4.9.1
+sphinx_rtd_theme==0.4.3

tox.ini

@@ -8,9 +8,11 @@ skipsdist = true
 basepython = python3
 deps = -r{toxinidir}/requirements.txt
 commands =
-    python -m mkdocs build --clean --site-dir _build/html --config-file mkdocs.yml
-    echo "Generated docs available in {toxinidir}/docs/_build/html"
+    rm -rf _build
+    sphinx-build -T -b html -D language=en doc _build/html
+    echo "Generated docs available in {toxinidir}/_build/html"
 whitelist_externals =
     echo
     git
-    sh
+    sh
+    rm