Add intersphinx config, docs on how to use (#703)

flyteorg · Feb 9, 2021 · 8905b86 · 8905b86
1 parent 034ad4f
commit 8905b86
Show file tree

Hide file tree

Showing 3 changed files with 142 additions and 25 deletions.
diff --git a/Makefile b/Makefile
@@ -20,16 +20,6 @@ end2end:
 end2end_execute:
 	@end2end/execute.sh
 
-# Use this target to build the rsts directory only. In order to build the entire flyte docs, use update_ref_docs && all_docs
-.PHONY: generate-local-docs
-generate-local-docs:
-	@docker run -t -v `pwd`:/base ghcr.io/nuclyde-io/docbuilder:e70cfafe3397c3b23d11183973c0f44e0024f025 sphinx-build -E -b html /base/rsts/. /base/_build
-
-# Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir
-.PHONY: generate-docs
-generate-docs: generate-dependent-repo-docs
-	@FLYTEKIT_VERSION=0.15.4 ./script/generate_docs.sh
-
 # updates referenced docs from other repositories (e.g. flyteidl, flytekit)
 .PHONY: generate-dependent-repo-docs
 generate-dependent-repo-docs:

diff --git a/rsts/conf.py b/rsts/conf.py
@@ -27,7 +27,6 @@
 # The full version, including alpha/beta/rc tags
 release = u'0.10.0'
 
-
 # -- General configuration ---------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -56,12 +55,12 @@
 ]
 
 extlinks = {
-   'propeller': ('https://github.com/lyft/flytepropeller/tree/master/%s', ''),
-   'stdlib':    ('https://github.com/lyft/flytestdlib/tree/master/%s', ''),
-   'kit':       ('https://github.com/lyft/flytekit/tree/master/%s', ''),
-   'plugins':   ('https://github.com/lyft/flyteplugins/tree/v0.1.4/%s', ''),
-   'idl':       ('https://github.com/lyft/flyteidl/tree/v0.14.1/%s', ''),
-   'admin':     ('https://github.com/lyft/flyteadmin/tree/master/%s', ''),
+    'propeller': ('https://github.com/lyft/flytepropeller/tree/master/%s', ''),
+    'stdlib': ('https://github.com/lyft/flytestdlib/tree/master/%s', ''),
+    'kit': ('https://github.com/lyft/flytekit/tree/master/%s', ''),
+    'plugins': ('https://github.com/lyft/flyteplugins/tree/v0.1.4/%s', ''),
+    'idl': ('https://github.com/lyft/flyteidl/tree/v0.14.1/%s', ''),
+    'admin': ('https://github.com/lyft/flyteadmin/tree/master/%s', ''),
 }
 
 # Add any paths that contain templates here, relative to this directory.
@@ -91,7 +90,6 @@
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
-
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -122,7 +120,6 @@
     "globaltoc_includehidden": False,
 }
 
-
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
@@ -144,13 +141,11 @@
 #
 html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]}
 
-
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'Flytedoc'
 
-
 # -- Options for LaTeX output ------------------------------------------------
 
 latex_elements = {
@@ -179,7 +174,6 @@
      u'Flyte Authors', 'manual'),
 ]
 
-
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
@@ -189,7 +183,6 @@
      [author], 1)
 ]
 
-
 # -- Options for Texinfo output ----------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
@@ -201,13 +194,25 @@
      'Miscellaneous'),
 ]
 
-
 # -- Extension configuration -------------------------------------------------
 
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+# intersphinx configuration. Uncomment the repeats with the local paths and update your username
+# to help with local development.
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "numpy": ("https://numpy.org/doc/stable", None),
+    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
+    "torch": ("https://pytorch.org/docs/master/", None),
+    "scipy": ("https://docs.scipy.org/doc/scipy/reference", None),
+    "matplotlib": ("https://matplotlib.org", None),
+    "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/master/", None),
+    # "flytekit": ("/Users/yourusername/go/src/github.com/flyteorg/flytekit/docs/build/html", None),
+    "flytectl": ("https://flytectl.readthedocs.io/en/latest/", None),
+    # "flytectl": ("/Users/yourusername/go/src/github.com/flyteorg/flytectl/docs/build/html", None),
+}
 
 # -- Options for todo extension ----------------------------------------------
 

diff --git a/rsts/contributor/docs/index.rst b/rsts/contributor/docs/index.rst
@@ -12,6 +12,7 @@ Documentation that the steps below will compile come from:
 * RST files generated from the Flyte IDL repository
 * RST files generated from the Flytekit Python SDK
 
+*****************************************************
 Building all documentation including dependent repos
 *****************************************************
 
@@ -28,6 +29,7 @@ What happens is:
 
 Please then commit the newly generated files before merging your PR.  In the future we will invest in CI to help with this.
 
+******************************************************************
 Building a local copy of documentation for RST modifications only
 ******************************************************************
 This can be used if one wants to quickly verify documentation updates in the rst files housed in the flyte repo. These are the main docs that flyte publishes.
@@ -38,8 +40,128 @@ Building all documentation can be a slow process. To speed up the iteration one
 This needs a local virtual environment with sphinx-build installed.
 
 
+*******
 Notes
 *******
 We aggregate the doc sources in a single ``index.rst`` file so that ``flytekit`` and the Flyte IDL HTML pages to be together in the same index/table of contents.
 
 There will be no separate Flyte Admin, Propeller, or Plugins documentation generation.  This is because we have a :ref:`contributor` guide and high level usage/architecture/runbook documentation should either there, or into the administrator's guide.
+
+
+***************
+Sphinx and RST
+***************
+
+Style
+=========
+
+Headers
+--------
+Typically, we try to follow these characters in this order for heading separation.
+
+.. code-block:: text
+
+    # with overline
+    * with overline
+    =
+    -
+    ^
+
+Intersphinx
+=============
+`Intersphinx <https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html>`__ is a plugin that all Flyte repos that build Sphinx documentation use for cross-referencing with each other. There's some good background information on it on these `slides <https://docs.google.com/presentation/d/1vkvsxp_64dhFuf7g3W8EHjK77lFOBhdeSg9VSA9-Ikc/>`__.
+
+Inventory File
+----------------
+When Sphinx runs, an inventory file gets created and is available alongside each repo's HTML pages. For example at ``https://readthedocs.org/projects/flytecookbook/objects.inv``. This file is a compressed inventory of all the sections, tags, etc. in the flyte cookbook documentation. This inventory file is what allows the intersphinx plugin to cross-link between projects.
+
+There is an open-source tool called ``sphobjinv`` that has managed to `reverse engineer these files <https://sphobjinv.readthedocs.io/en/stable/syntax.html>`__, and offers a CLI to help search for things inside them.
+
+Setup
+-------
+Installing ``sphobjinv`` and simple usage ::
+
+    $ pip install sphobjinv
+
+    # Using the CLI to query a hosted inventory file, note the -u switch
+    $ sphobjinv suggest https://flytekit.readthedocs.io/en/latest/ -u task
+
+    No inventory at provided URL.
+    Attempting "https://flytekit.readthedocs.io/en/latest/objects.inv" ...
+    Remote inventory found.
+
+    :py:function:`flytekit.task`
+    :std:doc:`tasks`
+    :std:doc:`tasks.extend`
+    :std:label:`tasks:tasks`
+
+    # Using the CLI to query a local file, useful when iterating locally
+    $ sphobjinv suggest ~/go/src/github.com/flyteorg/flytekit/docs/build/html/objects.inv task
+
+    :py:function:`flytekit.task`
+    :std:doc:`tasks`
+    :std:doc:`tasks.extend`
+    :std:label:`tasks:tasks`
+
+.. note::
+
+    Even though the ``sphobjinv`` CLI returns ``:py:function:...``, when actually creating a link you should just use ``:py:func:...``. See `this <https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html#cross-referencing-python-objects>`__.
+
+Linking Examples
+------------------
+In the ``conf.py`` file of each repo, there is an intersphinx mapping argument that looks something like this ::
+
+    intersphinx_mapping = {
+        "python": ("https://docs.python.org/3", None),
+        "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/master/", None),
+        ...
+    }
+
+This file is what tells the plugin where to look for these inventory files, and what project name to refer to each inventory file as. The project name is important because they're used when actually referencing something from the inventory.
+
+Here are some examples, first the code and then the link
+
+.. code-block:: text
+
+    Task: :std:doc:`flytekit:tasks`
+
+Task: :std:doc:`flytekit:tasks`
+
+-----
+
+.. code-block:: text
+
+    :std:doc:`Using custom words<flytekit:tasks>`
+
+:std:doc:`Using custom words<flytekit:tasks>`
+
+Python
+^^^^^^^
+Linking to Python elements changes based on what you're linking to. Check out this `section <https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html#cross-referencing-python-objects>`__. For instance linking to the ``task`` decorator in flytekit uses the ``func`` role.
+
+.. code-block:: text
+
+    Link to flytekit code :py:func:`flytekit:flytekit.task`
+
+Link to flytekit code :py:func:`flytekit:flytekit.task`
+
+Other elements use different Sphinx roles, here are some examples using Python core docs. ::
+
+    :py:mod:`Module <python:typing>`
+    :py:class:`Class <python:typing.Type>`
+    :py:data:`Data <python:typing.Callable>`
+    :py:func:`Function <python:typing.cast>`
+    :py:meth:`Method <python:pprint.PrettyPrinter.format>`
+
+
+:py:mod:`Module <python:typing>`
+
+:py:class:`Class <python:typing.Type>`
+
+:py:data:`Data <python:typing.Callable>`
+
+:py:func:`Function <python:typing.cast>`
+
+:py:meth:`Method <python:pprint.PrettyPrinter.format>`
+
+