Revised sections of the SDC documentation; getting rid of autodoc/autosummary/autolabel; scripts to traverse module classes and parse docstrings (#317)

Author: samaid

docs/source/apireference.rst

@@ -1,11 +1,11 @@
 .. _apireference:
 
-API Reference: Supported Pandas APIs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+API Reference
+~~~~~~~~~~~~~
 
-This page gives an overview of all Pandas APIs supported currently by Intel® SDC
+This page gives an overview of all Pandas APIs supported currently by Intel® Scalable Dataframe Compiler
 
-.. toctree::
-   :maxdepth: 2
-   
-   series
+.. autosummary::
+    :toctree: _autosummary
+
+    pandas.Series

docs/source/compilation.rst

@@ -1,16 +1,16 @@
 .. _compilation:
 .. include:: ./ext_links.txt
 
-Compiling With Intel® SDC
-=========================
+Compiling With Intel® Scalable Dataframe Compiler
+=================================================
 
 .. todo::
      Basic compilation controls. What can be compiled and what cannot. How to work around compilation issues.
      References to relevant discussion in `Numba*`_. Specifics for Series, Dataframes, and other hpat specific
      data structures
 
 What if I get a compilation error
-=================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 There are a few reasons why Intel SDC cannot compile your code out-of-the-box.
 

docs/source/conf.py

@@ -51,11 +51,11 @@
 
 if not sdc_doc_no_api_ref:
     try:
-        import sdc  # TODO: Rename hpat module name to sdc
+        import sdc
     except ImportError:
-        print('IMPORT EXCEPTION: Cannot import sdc. ')
+        print('IMPORT EXCEPTION: Cannot import SDC. ')
         print('Documentation generator for API Reference for a given module expects that module '
-              'to be installed. Use conda/pip install hpat to install it prior to using API Reference generation')
+              'to be installed. Use conda/pip install SDC to install it prior to using API Reference generation')
         print('If you want to disable API Reference generation, set the environment variable SDC_DOC_NO_API_REF=1')
 
         raise
@@ -77,15 +77,15 @@
 # ones.
 extensions = [
     'sphinx.ext.todo',
-    'sphinx.ext.autosummary',
+#    'sphinx.ext.autosummary',
     'sphinx.ext.intersphinx',
-    'sphinx.ext.autodoc',
+#    'sphinx.ext.autodoc',
     'sphinx.ext.extlinks',
     'sphinx.ext.githubpages',
     'sphinx.ext.napoleon',
-    'sphinx.ext.autosectionlabel',
-    'sphinx.ext.graphviz',
-    'sphinx.ext.coverage'
+#    'sphinx.ext.autosectionlabel',
+#    'sphinx.ext.graphviz',
+#    'sphinx.ext.coverage'
 ]
 
 
@@ -149,94 +149,15 @@
 
 
 # -- Auto-section label configuration -----------------------------------------------
-autosectionlabel_prefix_document = True
+#autosectionlabel_prefix_document = True
 
 
 # -- Autodoc configuration ----------------------------------------------------------
-autodoc_docstring_signature = True
+#autodoc_docstring_signature = True
 
 
 # -- Auto-summary configuration -----------------------------------------------------
-autosummary_generate = True
+#autosummary_generate = True
 
 # -- Prepend module name to an object name or not -----------------------------------
 add_module_names = False
-
-
-"""
-import API_Doc
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-# -- Options for HTMLHelp output ------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'HPATdoc'
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_engine = 'pdflatex'
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (master_doc, 'HPAT.tex', 'HPAT Documentation',
-     'Intel', 'manual'),
-]
-
-pdf_documents = [
-    ('index', u'HPATDocumentation', u'HPAT Documentation', u'Rujal Desai'),
-]
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'sdc', 'HPAT Documentation',
-     [author], 1)
-]
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (master_doc, 'HPAT', 'HPAT Documentation',
-     author, 'HPAT', 'One line description of project.',
-     'Miscellaneous'),
-]
-
-numfig = True
-
-# configuration for intersphinx
-intersphinx_mapping = {
-    'python': ('https://docs.python.org/', None),
-    'numba': ('https://numba.pydata.org/numba-doc/dev', None),
-    'numpy': ('http://docs.scipy.org/doc/numpy', None),
-    'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None),
-}
-"""

docs/source/info.py

@@ -0,0 +1,135 @@
+# Copyright 2015: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import re
+import sys
+
+PARAM_OR_RETURNS_REGEX = re.compile(":(?:param|returns)")
+RETURNS_REGEX = re.compile(":returns: (?P<doc>.*)", re.S)
+PARAM_REGEX = re.compile(":param (?P<name>[\*\w]+): (?P<doc>.*?)"
+                         "(?:(?=:param)|(?=:return)|(?=:raises)|\Z)", re.S)
+
+def trim(docstring):
+    """trim function from PEP-257"""
+    if not docstring:
+        return ""
+    # Convert tabs to spaces (following the normal Python rules)
+    # and split into a list of lines:
+    lines = docstring.expandtabs().splitlines()
+    # Determine minimum indentation (first line doesn't count):
+    indent = sys.maxsize
+    for line in lines[1:]:
+        stripped = line.lstrip()
+        if stripped:
+            indent = min(indent, len(line) - len(stripped))
+    # Remove indentation (first line is special):
+    trimmed = [lines[0].strip()]
+    if indent < sys.maxsize:
+        for line in lines[1:]:
+            trimmed.append(line[indent:].rstrip())
+    # Strip off trailing and leading blank lines:
+    while trimmed and not trimmed[-1]:
+        trimmed.pop()
+    while trimmed and not trimmed[0]:
+        trimmed.pop(0)
+
+    # Current code/unittests expects a line return at
+    # end of multiline docstrings
+    # workaround expected behavior from unittests
+    if "\n" in docstring:
+        trimmed.append("")
+
+    # Return a single string:
+    return "\n".join(trimmed)
+
+
+def reindent(string):
+    return "\n".join(l.strip() for l in string.strip().split("\n"))
+
+
+def parse_docstring(docstring):
+    """Parse the docstring into its components.
+    :returns: a dictionary of form
+              {
+                  "short_description": ...,
+                  "long_description": ...,
+                  "params": [{"name": ..., "doc": ...}, ...],
+                  "returns": ...
+              }
+    """
+
+    short_description = long_description = returns = ""
+    params = []
+
+    if docstring:
+        docstring = trim(docstring)
+
+        lines = docstring.split("\n", 1)
+        short_description = lines[0]
+
+        if len(lines) > 1:
+            long_description = lines[1].strip()
+
+            params_returns_desc = None
+
+            match = PARAM_OR_RETURNS_REGEX.search(long_description)
+            if match:
+                long_desc_end = match.start()
+                params_returns_desc = long_description[long_desc_end:].strip()
+                long_description = long_description[:long_desc_end].rstrip()
+
+            if params_returns_desc:
+                params = [
+                   {"name": name, "doc": trim(doc)}
+                   for name, doc in PARAM_REGEX.findall(params_returns_desc)
+                ]
+
+                match = RETURNS_REGEX.search(params_returns_desc)
+                if match:
+                    returns = reindent(match.group("doc"))
+
+    return {
+        "short_description": short_description,
+        "long_description": long_description,
+        "params": params,
+        "returns": returns
+    }
+
+
+class InfoMixin(object):
+
+    @classmethod
+    def _get_doc(cls):
+        """Return documentary of class
+        By default it returns docstring of class, but it can be overridden
+        for example for cases like merging own docstring with parent
+        """
+        return cls.__doc__
+
+    @classmethod
+    def get_info(cls):
+        doc = parse_docstring(cls._get_doc())
+
+        return {
+            "name": cls.get_name(),
+            "platform": cls.get_platform(),
+            "platform": cls.get_platform(),
+            "module": cls.__module__,
+            "title": doc["short_description"],
+            "description": doc["long_description"],
+            "parameters": doc["params"],
+            "schema": getattr(cls, "CONFIG_SCHEMA", None),
+            "returns": doc["returns"]
+        }