From e9f6795bb64196ae2d0055de94e50bace4bfe1e8 Mon Sep 17 00:00:00 2001
From: muecelee <ialreadygetenough@spam.com>
Date: Sun, 18 Oct 2020 10:16:45 +0200
Subject: [PATCH 1/6] ENH: Add support for reST-style docstrings

---
 pdoc/html_helpers.py              | 100 +++++++++++++++++++++++++++---
 pdoc/test/__init__.py             |  30 +++++++++
 pdoc/test/example_pkg/__init__.py |  22 +++++++
 3 files changed, 144 insertions(+), 8 deletions(-)

diff --git a/pdoc/html_helpers.py b/pdoc/html_helpers.py
index e273b824..23de8085 100644
--- a/pdoc/html_helpers.py
+++ b/pdoc/html_helpers.py
@@ -265,6 +265,71 @@ def googledoc_sections(match):
                           r'((?:\n?(?: {2,}.*|$))+)', re.MULTILINE).sub(googledoc_sections, text)
         return text
 
+    @staticmethod
+    def reST(text: str) -> str:
+        """
+        Convert `text` in reST-style docstring format to Markdown
+        to be further converted later.
+        """
+        def reST_sections(match) -> str:
+            # E.g. for ":param arg1: Text" tag is "param", name is "arg1", and body is "Text"
+            tag, name, body = match.groups('')
+            body = textwrap.dedent(body)
+
+            type_ = None
+            nonlocal active_section
+            active_section_changed = False
+
+            if tag in ['type', 'rtype']:
+                return ''
+            elif tag in ('param', 'parameter', 'arg', 'argument', 'key', 'keyword'):
+                type_ = parameter_types.get(name, None)
+
+                if active_section != 'Args':
+                    active_section = 'Args'
+                    active_section_changed = True
+            elif tag in ('return', 'returns'):
+                if len(return_types) > 0:
+                    type_ = return_types.pop()
+
+                if active_section != 'Returns':
+                    active_section = 'Returns'
+                    active_section_changed = True
+            elif tag in ('raise', 'raises'):
+                if active_section != 'Raises':
+                    active_section = 'Raises'
+                    active_section_changed = True
+
+            if name or type_:
+                text = _ToMarkdown._deflist(*_ToMarkdown._fix_indent(name, type_, body))
+            else:
+                _, _, body = _ToMarkdown._fix_indent(name, type_, body)
+                text = f':   {body}'
+
+            if active_section_changed:
+                text = f'\n{active_section}:\n-----=\n{text}'
+            else:
+                text = f'\n{text}'
+
+            return text
+
+        regex = re.compile(r'^:(\S+)(?:\s(\S+?))?:((?:\n?(?: .*|$))+)', re.MULTILINE)
+
+        # Get all parameter and return types beforehand, to then use them when substituting
+        # the sections
+        parameter_types = {}
+        return_types = []
+        for tag, name, body in regex.findall(text):
+            if tag == 'type':
+                parameter_types[name] = body.strip()
+            elif tag == 'rtype':
+                return_types.append(body.strip())
+
+        active_section = None  # Keep track of the currently active section (e.g. Args, Returns)
+        text = regex.sub(reST_sections, text)
+
+        return text
+
     @staticmethod
     def _admonition(match, module=None, limit_types=None):
         indent, type, value, text = match.groups()
@@ -406,8 +471,9 @@ def to_html(text: str, *,
             latex_math: bool = False):
     """
     Returns HTML of `text` interpreted as `docformat`. `__docformat__` is respected
-    if present, otherwise Numpydoc and Google-style docstrings are assumed,
-    as well as pure Markdown.
+    if present, otherwise it is inferred whether it's reST-style, or Numpydoc
+    and Google-style docstrings. Pure Markdown and reST directives are also assumed
+    and processed if docformat has not been specified.
 
     `module` should be the documented module (so the references can be
     resolved) and `link` is the hyperlinking function like the one in the
@@ -430,20 +496,35 @@ def to_markdown(text: str, *,
                 module: pdoc.Module = None, link: Callable[..., str] = None):
     """
     Returns `text`, assumed to be a docstring in `docformat`, converted to markdown.
-    `__docformat__` is respected
-    if present, otherwise Numpydoc and Google-style docstrings are assumed,
-    as well as pure Markdown.
+    `__docformat__` is respected if present, otherwise it is inferred whether it's
+     reST-style, or Numpydoc and Google-style docstrings. Pure Markdown and reST directives
+     are also assumed and processed if docformat has not been specified.
 
     `module` should be the documented module (so the references can be
     resolved) and `link` is the hyperlinking function like the one in the
     example template.
     """
     if not docformat:
-        docformat = str(getattr(getattr(module, 'obj', None), '__docformat__', 'numpy,google '))
+        docformat = str(getattr(getattr(module, 'obj', None), '__docformat__', ''))
+
+        # Infer docformat if it hasn't been specified
+        if docformat == '':
+            reST_tags = ['param', 'arg', 'type', 'raise', 'except', 'return', 'rtype']
+            reST_regex = fr'^:(?:{"|".join(reST_tags)}).*?:'
+            found_reST_tags = re.findall(reST_regex, text, re.MULTILINE)
+
+            # Assume reST-style docstring if any of the above specified tags is present at the beginning of a line.
+            # Could make this more robust, e.g., by checking against the amount of found google or numpy tags
+            if len(found_reST_tags) > 0:
+                docformat = 'reST '
+            else:
+                docformat = 'numpy,google '
+
         docformat, *_ = docformat.lower().split()
-    if not (set(docformat.split(',')) & {'', 'numpy', 'google'}):
+
+    if not (set(docformat.split(',')) & {'', 'numpy', 'google', 'rest'}):
         warn('__docformat__ value {!r} in module {!r} not supported. '
-             'Supported values are: numpy, google.'.format(docformat, module))
+             'Supported values are: numpy, google, reST.'.format(docformat, module))
         docformat = 'numpy,google'
 
     with _fenced_code_blocks_hidden(text) as result:
@@ -462,6 +543,9 @@ def to_markdown(text: str, *,
         if 'numpy' in docformat:
             text = _ToMarkdown.numpy(text)
 
+        if 'rest' in docformat:
+            text = _ToMarkdown.reST(text)
+
         if module and link:
             # Hyperlink markdown code spans not within markdown hyperlinks.
             # E.g. `code` yes, but not [`code`](...). RE adapted from:
diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
index 4c482192..946f9f34 100644
--- a/pdoc/test/__init__.py
+++ b/pdoc/test/__init__.py
@@ -1403,6 +1403,36 @@ def test_google(self):
         html = to_html(text, module=self._module, link=self._link)
         self.assertEqual(html, expected)
 
+    def test_reST(self):
+        expected = '''<p>Summary line.</p>
+<h2 id="args">Args:</h2>
+<dl>
+<dt><strong><code>arg1</code></strong> :&ensp;<code>int</code></dt>
+<dd>Text1</dd>
+<dt><strong><code>arg2</code></strong> :&ensp;<code>Optional[List[Tuple[str]]]</code></dt>
+<dd>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
+ diam nonumy eirmod tempor invidunt</dd>
+<dt><strong><code>arg_arg_3</code></strong> :&ensp;<code>Dict[int, Dict[str, Any]]</code></dt>
+<dd>Y:=H^T<em>X!@#$%^&amp;&amp;</em>()_[]{}';'::</dd>
+</dl>
+<h2 id="returns">Returns:</h2>
+<dl>
+<dt><code>bool</code></dt>
+<dd>True. Or False. Depends</dd>
+<dd>Now with more "s"</dd>
+</dl>
+<h2 id="raises">Raises:</h2>
+<dl>
+<dt><strong><code>Exception</code></strong></dt>
+<dd>Raised occasionally</dd>
+<dt><strong><code>ZeroDivisionError</code></strong></dt>
+<dd>You know why and when</dd>
+</dl>'''
+        text = inspect.getdoc(self._docmodule.reST)
+        html = to_html(text, module=self._module, link=self._link)
+
+        self.assertEqual(html, expected)
+
     def test_doctests(self):
         expected = '''<p>Need an intro paragrapgh.</p>
 <pre><code>&gt;&gt;&gt; Then code is indented one level
diff --git a/pdoc/test/example_pkg/__init__.py b/pdoc/test/example_pkg/__init__.py
index 4a8aeef9..559cbb0c 100644
--- a/pdoc/test/example_pkg/__init__.py
+++ b/pdoc/test/example_pkg/__init__.py
@@ -298,6 +298,25 @@ def doctests(self):
             Exception: something went wrong
         """
 
+    def reST(self):
+        """
+        Summary line.
+
+        :param arg1: Text1
+        :parameter arg2: Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
+            diam nonumy eirmod tempor invidunt
+        :arg arg_arg_3: Y:=H^T*X!@#$%^&&*()_[]{}';'::
+        :type arg1: int
+        :type arg2: Optional[List[Tuple[str]]]
+        :type arg_arg_3: Dict[int, Dict[str, Any]]
+        :return: True. Or False. Depends
+        :rtype: bool
+
+        :returns: Now with more "s"
+        :raise Exception: Raised occasionally
+        :raises ZeroDivisionError: You know why and when
+        """
+
     def reST_directives(self):
         """
         .. todo::
@@ -345,6 +364,9 @@ def reST_directives(self):
 doctests = Docformats.doctests
 
 
+reST = Docformats.reST
+
+
 reST_directives = Docformats.reST_directives
 
 

From 893e4bb6e85398c80daa5ee98d2cd2b10b337109 Mon Sep 17 00:00:00 2001
From: muecelee <ialreadygetenough@spam.com>
Date: Wed, 21 Oct 2020 17:52:27 +0200
Subject: [PATCH 2/6] Change reST docstring processing to use docutils

Change "restructuredtext" to "epytext" in test_docformat() since
restructuredtext is now (somewhat) supported
Add trailing spaces to reST tags at docstring type inference
---
 pdoc/html_helpers.py              | 214 +++++++++++++++++++++---------
 pdoc/test/__init__.py             | 148 ++++++++++++++++-----
 pdoc/test/example_pkg/__init__.py |  66 ++++++++-
 setup.py                          |   1 +
 4 files changed, 333 insertions(+), 96 deletions(-)

diff --git a/pdoc/html_helpers.py b/pdoc/html_helpers.py
index 23de8085..a68d0af1 100644
--- a/pdoc/html_helpers.py
+++ b/pdoc/html_helpers.py
@@ -12,6 +12,11 @@
 from typing import Callable, Match
 from warnings import warn
 import xml.etree.ElementTree as etree
+import docutils.nodes
+import docutils.core
+import collections
+from typing import *
+
 
 import markdown
 from markdown.inlinepatterns import InlineProcessor
@@ -266,69 +271,151 @@ def googledoc_sections(match):
         return text
 
     @staticmethod
-    def reST(text: str) -> str:
-        """
-        Convert `text` in reST-style docstring format to Markdown
-        to be further converted later.
+    def _reST_string_to_html(text: str) -> str:
+        """Convert reST text to html using docutils
+
+        :param text: The text to convert
+        :returns: The generated html
         """
-        def reST_sections(match) -> str:
-            # E.g. for ":param arg1: Text" tag is "param", name is "arg1", and body is "Text"
-            tag, name, body = match.groups('')
-            body = textwrap.dedent(body)
+        html = docutils.core.publish_parts(text, writer_name='html')['html_body']
 
-            type_ = None
-            nonlocal active_section
-            active_section_changed = False
-
-            if tag in ['type', 'rtype']:
-                return ''
-            elif tag in ('param', 'parameter', 'arg', 'argument', 'key', 'keyword'):
-                type_ = parameter_types.get(name, None)
-
-                if active_section != 'Args':
-                    active_section = 'Args'
-                    active_section_changed = True
-            elif tag in ('return', 'returns'):
-                if len(return_types) > 0:
-                    type_ = return_types.pop()
-
-                if active_section != 'Returns':
-                    active_section = 'Returns'
-                    active_section_changed = True
-            elif tag in ('raise', 'raises'):
-                if active_section != 'Raises':
-                    active_section = 'Raises'
-                    active_section_changed = True
-
-            if name or type_:
-                text = _ToMarkdown._deflist(*_ToMarkdown._fix_indent(name, type_, body))
-            else:
-                _, _, body = _ToMarkdown._fix_indent(name, type_, body)
-                text = f':   {body}'
+        # Remove the document tag and return
+        return html[23:-8]
 
-            if active_section_changed:
-                text = f'\n{active_section}:\n-----=\n{text}'
-            else:
-                text = f'\n{text}'
+    @staticmethod
+    def _reST_node_to_html(node: docutils.nodes.Node,
+                          doctree: docutils.nodes.document) -> str:
+        """Not all nodes in the doctree provide their reST source or at least the
+        starting line in the reST source. This method simply copies the document
+        tree and removes all but the node to then publish it.
+
+        :node: The node to publish. Must be a child of `doctree` itself
+        :doctree: The document having `node` as a child node
+        :return: The generated html for this node
+        """
+        # Remove all but the given node from the doctree
+        children_copy = doctree.children
+        doctree.children = [doctree.children[doctree.index(node)]]
+
+        # Generate the html for this node/doctree
+        html = docutils.core.publish_from_doctree(doctree, writer_name='html5').decode('utf-8')
 
-            return text
+        # Restore the doctree
+        doctree.children = children_copy
 
-        regex = re.compile(r'^:(\S+)(?:\s(\S+?))?:((?:\n?(?: .*|$))+)', re.MULTILINE)
+        # Return only the relevant part of the html
+        return re.search(r"<div class=\"document\">(.+)<\/div>", html, re.DOTALL).group(1).strip()
 
-        # Get all parameter and return types beforehand, to then use them when substituting
-        # the sections
-        parameter_types = {}
-        return_types = []
-        for tag, name, body in regex.findall(text):
-            if tag == 'type':
-                parameter_types[name] = body.strip()
+    @staticmethod
+    def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
+                                                       docutils.nodes.docinfo]) -> str:
+        """Processes a docutils field list and converts it to markdown.
+        Args, Vars, Returns, and Raises sections are predefined, other sections
+        will be created corresponding to their field names.
+
+        :param field_list: A docutils field list to convert. Can also be a docinfo
+            in case, e.g., only :returns: is specified without any summary text
+        :returns: The generated Markdown. However, it is not pure Markdown, as
+            the field descriptions have been processed with `docutils.process_string` - they
+            therefore are already converted to HTML
+        """
+        # Sort the field list so that types come last  - in case someone defines first the type then
+        # the parameter
+        field_list.children.sort(key=lambda field: 'type' in field[0].rawsource.split()[0])
+
+        # Predefined sections for the generated markdown
+        tags_to_section_map = {
+            ('param', 'parameter', 'arg', 'argument', 'key', 'keyword'): 'Args',
+            ('var', 'ivar', 'cvar'): 'Vars',
+            ('return', 'returns'): 'Returns',
+            ('raise', 'raises'): 'Raises'
+        }
+        sections = collections.OrderedDict([('Args', []), ('Vars', []),
+                                            ('Returns', []), ('Raises', [])])
+
+        # Process the fields
+        for field in field_list:
+            field_name = field.children[0]
+            field_body = field.children[1]
+
+            # Split the field name into its components
+            split = field_name.rawsource.split()
+            tag = split[0]
+            type_ = split[1] if len(split) == 3 else None
+            name = split[2] if len(split) == 3 else split[1] if len(split) == 2 else None
+
+            # Fill the sections
+            try:
+                section = [section for tags, section in tags_to_section_map.items() if tag in tags][0]
+            except IndexError:  # Field is not corresponding to a predefined section like Args
+                section = None
+
+            if section is not None:
+                sections[section].append({'name': name,
+                                          'type': type_,
+                                          'body': _ToMarkdown._reST_string_to_html(
+                                              field_body.rawsource)})
             elif tag == 'rtype':
-                return_types.append(body.strip())
+                # Set the return type. Assumes that at most one :return: has been specified
+                try:
+                    sections['Returns'][0]['type'] = field_body.rawsource
+                except IndexError:  # Only return type is specified
+                    sections['Returns'].append({'name': None,
+                                                'type': field_body.rawsource.strip(),
+                                                'body': ''})
+            elif 'type' in tag:
+                section = 'Vars' if tag == 'vartype' else 'Args'
+                try:
+                    param_or_var = [x for x in sections[section] if x['name'] == name][0]
+                    param_or_var['type'] = field_body.rawsource.strip()
+                except IndexError:  # Only parameter (or variable) type is specified
+                    sections[section].append({'name': name,
+                                              'type': field_body.rawsource.strip(),
+                                              'body': ''})
+            elif tag == 'meta':
+                pass  # Meta fields should be excluded from the final output
+            else:
+                # Generate sections for tags not covered yet
+                section = sections.get(tag, [])
+                section.append({'name': name,
+                                'type': type_,
+                                'body': _ToMarkdown._reST_string_to_html(field_body.rawsource)})
+                sections[tag] = section
+
+        # Generate the markdown for this field list
+        markdown = []
+        for section, fields in sections.items():
+            if len(fields) > 0:  # Skip empty sections
+                markdown.append(f'{section}:\n-----=')
+
+                for field in fields:
+                    field['body'] = field['body'].replace('\n', '\n  ')  # Allow for proper indentation
+                    if field['name'] or field['type']:
+                        markdown.append(_ToMarkdown._deflist(*_ToMarkdown._fix_indent(field['name'],
+                                                                                      field['type'],
+                                                                                      field['body'])))
+                    else:  # For fields with no name or type (e.g. Returns without type specification)
+                        text = _ToMarkdown._fix_indent(field['name'], field['type'], field['body'])[2]
+                        markdown.append(f':   {text}')
+
+        return '\n'.join(markdown)
 
-        active_section = None  # Keep track of the currently active section (e.g. Args, Returns)
-        text = regex.sub(reST_sections, text)
+    @staticmethod
+    def reST(text: str) -> str:
+        """
+        Convert `text` in reST-style docstring format to Markdown - with embedded html
+        for paragraphs and field descriptions - to be further converted later.
+        """
+        doctree = docutils.core.publish_doctree(text)
 
-        return text
+        generated_markdown = []
+        for section in doctree:
+            if section.tagname in ('field_list', 'docinfo'):
+                generated_markdown.append(_ToMarkdown._reST_field_list_to_markdown(section))
+            else:
+                generated_markdown.append(_ToMarkdown._reST_node_to_html(section, doctree))
+
+        return '\n'.join(generated_markdown)
 
     @staticmethod
     def _admonition(match, module=None, limit_types=None):
@@ -509,41 +596,40 @@ def to_markdown(text: str, *,
 
         # Infer docformat if it hasn't been specified
         if docformat == '':
-            reST_tags = ['param', 'arg', 'type', 'raise', 'except', 'return', 'rtype']
+            reST_tags = ['param ', 'arg ', 'type ', 'raise ', 'except ', 'return', 'rtype']
             reST_regex = fr'^:(?:{"|".join(reST_tags)}).*?:'
             found_reST_tags = re.findall(reST_regex, text, re.MULTILINE)
 
             # Assume reST-style docstring if any of the above specified tags is present at the beginning of a line.
             # Could make this more robust, e.g., by checking against the amount of found google or numpy tags
-            if len(found_reST_tags) > 0:
-                docformat = 'reST '
-            else:
-                docformat = 'numpy,google '
+            docformat = 'restructuredtext ' if len(found_reST_tags) > 0 else 'numpy,google '
 
         docformat, *_ = docformat.lower().split()
 
-    if not (set(docformat.split(',')) & {'', 'numpy', 'google', 'rest'}):
+    if not (set(docformat.split(',')) & {'', 'numpy', 'google', 'restructuredtext'}):
         warn('__docformat__ value {!r} in module {!r} not supported. '
-             'Supported values are: numpy, google, reST.'.format(docformat, module))
+             'Supported values are: numpy, google, restructuredtext.'.format(docformat, module))
         docformat = 'numpy,google'
 
     with _fenced_code_blocks_hidden(text) as result:
         text = result[0]
 
-        text = _ToMarkdown.admonitions(text, module)
+        if 'restructuredtext' not in docformat:  # Will be handled by docutils
+            text = _ToMarkdown.admonitions(text, module)
 
         if 'google' in docformat:
             text = _ToMarkdown.google(text)
 
         text = _ToMarkdown.doctests(text)
-        text = _ToMarkdown.raw_urls(text)
+        if 'restructuredtext' not in docformat:  # Will be handled by docutils
+            text = _ToMarkdown.raw_urls(text)
 
         # If doing both, do numpy after google, otherwise google-style's
         # headings are incorrectly interpreted as numpy params
         if 'numpy' in docformat:
             text = _ToMarkdown.numpy(text)
 
-        if 'rest' in docformat:
+        if 'restructuredtext' in docformat:
             text = _ToMarkdown.reST(text)
 
         if module and link:
diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
index 946f9f34..4bd72343 100644
--- a/pdoc/test/__init__.py
+++ b/pdoc/test/__init__.py
@@ -234,7 +234,7 @@ def test_html_ref_links(self):
 
     def test_docformat(self):
         with self.assertWarns(UserWarning) as cm,\
-                run_html(EXAMPLE_MODULE, config='docformat="restructuredtext"'):
+                run_html(EXAMPLE_MODULE, config='docformat="epytext"'):
             self._basic_html_assertions()
         self.assertIn('numpy', cm.warning.args[0])
 
@@ -1403,36 +1403,6 @@ def test_google(self):
         html = to_html(text, module=self._module, link=self._link)
         self.assertEqual(html, expected)
 
-    def test_reST(self):
-        expected = '''<p>Summary line.</p>
-<h2 id="args">Args:</h2>
-<dl>
-<dt><strong><code>arg1</code></strong> :&ensp;<code>int</code></dt>
-<dd>Text1</dd>
-<dt><strong><code>arg2</code></strong> :&ensp;<code>Optional[List[Tuple[str]]]</code></dt>
-<dd>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
- diam nonumy eirmod tempor invidunt</dd>
-<dt><strong><code>arg_arg_3</code></strong> :&ensp;<code>Dict[int, Dict[str, Any]]</code></dt>
-<dd>Y:=H^T<em>X!@#$%^&amp;&amp;</em>()_[]{}';'::</dd>
-</dl>
-<h2 id="returns">Returns:</h2>
-<dl>
-<dt><code>bool</code></dt>
-<dd>True. Or False. Depends</dd>
-<dd>Now with more "s"</dd>
-</dl>
-<h2 id="raises">Raises:</h2>
-<dl>
-<dt><strong><code>Exception</code></strong></dt>
-<dd>Raised occasionally</dd>
-<dt><strong><code>ZeroDivisionError</code></strong></dt>
-<dd>You know why and when</dd>
-</dl>'''
-        text = inspect.getdoc(self._docmodule.reST)
-        html = to_html(text, module=self._module, link=self._link)
-
-        self.assertEqual(html, expected)
-
     def test_doctests(self):
         expected = '''<p>Need an intro paragrapgh.</p>
 <pre><code>&gt;&gt;&gt; Then code is indented one level
@@ -1465,6 +1435,121 @@ def test_doctests(self):
         html = to_html(text, module=self._module, link=self._link)
         self.assertEqual(html, expected)
 
+    def test_reST(self):
+        expected = '''<p>Summary line.</p>
+<p>Some stuff to test like <a class="reference external" href="http://www.python.org">http://www.python.org</a> or <a class="reference external" href="http://www.python.org">link_text</a>.
+Also <em>italic</em> and <strong>bold</strong>. And lists:</p>
+<ul class="simple">
+<li><p>1</p></li>
+<li><p>2</p></li>
+</ul>
+<ol class="arabic simple">
+<li><p>Item</p></li>
+<li><p>Item</p></li>
+</ol>
+<h2 id="args">Args:</h2>
+<dl>
+<dt><strong><code>arg1</code></strong> :&ensp;<code>int</code></dt>
+<dd><p>Text1</p></dd>
+<dt><strong><code>arg2</code></strong> :&ensp;<code>Optional[List[Tuple[str]]]</code></dt>
+<dd><p>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
+diam nonumy eirmod tempor invidunt</p></dd>
+<dt><strong><code>arg_arg_3</code></strong> :&ensp;<code>Dict[int, Dict[str, Any]]</code></dt>
+<dd><p>Y:=H^T<em>X!&#64;#$%^&amp;&amp;</em>()_[]{}';'::{[( :param just_in_case:</p></dd>
+<dt><strong><code>another_parameter</code></strong> :&ensp;<code>str</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<h2 id="vars">Vars:</h2>
+<dl>
+<dt><strong><code>x</code></strong></dt>
+<dd><p>Description of variable x</p></dd>
+<dt><strong><code>y</code></strong> :&ensp;<code>List[bool]</code></dt>
+<dd><p>Description of variable y</p></dd>
+<dt><strong><code>z</code></strong> :&ensp;<code>str</code></dt>
+<dd><p>Descriptions can also be placed in a new line.</p>
+<p>And span multiple lines.</p></dd>
+</dl>
+<h2 id="returns">Returns:</h2>
+<dl>
+<dt><code>bool</code></dt>
+<dd><p>True. Or False. Depends</p></dd>
+</dl>
+<p>A paragraph to split the field list into two.</p>
+<h2 id="returns_1">Returns:</h2>
+<dl>
+<dd>
+<dl>
+<dt><p>Now with more &quot;s&quot;</p></dt>
+<dt>Raises:</dt>
+<dt>-----=</dt>
+<dt><strong><code>Exception</code></strong></dt>
+<dd><p>Raised occasionally</p></dd>
+</dl>
+</dd>
+<dt><strong><code>ZeroDivisionError</code></strong></dt>
+<dd><p>You know why and when</p></dd>
+</dl>
+<p>Some more tests below:</p>
+<h2 id="args_1">Args:</h2>
+<dl>
+<dt><strong><code>z</code></strong> :&ensp;<code>str</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<h2 id="vars_1">Vars:</h2>
+<dl>
+<dt><strong><code>x</code></strong> :&ensp;<code>int</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<h2 id="returns_2">Returns:</h2>
+<dl>
+<dt><code>int</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<p>And now for some other stuff</p>
+<div class="admonition admonition-todo">
+<p class="admonition-title">TODO</p>
+<p>Create something.</p>
+</div>
+<div class="admonition admonition-example">
+<p class="admonition-title">Example</p>
+<p>Image shows something.</p>
+<img alt="https://www.debian.org/logos/openlogo-nd-100.png" src="https://www.debian.org/logos/openlogo-nd-100.png" />
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>Can only nest admonitions two levels.</p>
+</div>
+</div>
+<p><img alt="https://www.debian.org/logos/openlogo-nd-100.png" src="https://www.debian.org/logos/openlogo-nd-100.png" /></p>
+<p>Now you know.</p>
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>Some warning
+lines.</p>
+</div>
+<ul>
+<li><p>Describe some func in a list
+across multiple lines:</p>
+<blockquote>
+<div class="admonition admonition-deprecated-since-3-1">
+<p class="admonition-title">Deprecated since 3.1</p>
+<p>Use <cite>spam</cite> instead.</p>
+</div>
+<div class="admonition admonition-added-in-version-2-5">
+<p class="admonition-title">Added in version 2.5</p>
+<p>The <em>spam</em> parameter.</p>
+</div>
+</blockquote>
+</li>
+</ul>
+<div class="admonition caution">
+<p class="admonition-title">Caution!</p>
+<p>Don't touch this!</p>
+</div>'''
+        text = inspect.getdoc(self._docmodule.reST)
+        html = to_html(text, module=self._module, link=self._link)
+
+        self.assertEqual(html, expected)
+
     def test_reST_directives(self):
         expected = '''<div class="admonition todo">
 <p class="admonition-title">TODO</p>
@@ -1506,6 +1591,7 @@ def test_reST_directives(self):
 </div>'''
         text = inspect.getdoc(self._docmodule.reST_directives)
         html = to_html(text, module=self._module, link=self._link)
+
         self.assertEqual(html, expected)
 
     def test_reST_include(self):
diff --git a/pdoc/test/example_pkg/__init__.py b/pdoc/test/example_pkg/__init__.py
index 559cbb0c..1176bfb4 100644
--- a/pdoc/test/example_pkg/__init__.py
+++ b/pdoc/test/example_pkg/__init__.py
@@ -302,19 +302,83 @@ def reST(self):
         """
         Summary line.
 
+        Some stuff to test like http://www.python.org or `link_text <http://www.python.org>`_.
+        Also *italic* and **bold**. And lists:
+
+        * 1
+        * 2
+
+        #. Item
+        #. Item
+
         :param arg1: Text1
         :parameter arg2: Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
             diam nonumy eirmod tempor invidunt
-        :arg arg_arg_3: Y:=H^T*X!@#$%^&&*()_[]{}';'::
+        :arg arg_arg_3: Y:=H^T*X!@#$%^&&*()_[]{}';'::{[( :param just_in_case:
+        :key str another_parameter:
         :type arg1: int
         :type arg2: Optional[List[Tuple[str]]]
         :type arg_arg_3: Dict[int, Dict[str, Any]]
+        :var x: Description of variable x
+        :ivar y: Description of variable y
+        :cvar str z:
+            Descriptions can also be placed in a new line.
+
+            And span multiple lines.
+        :vartype y: List[bool]
         :return: True. Or False. Depends
         :rtype: bool
+        :meta: This should not be displayed in the generated output document
+
+        A paragraph to split the field list into two.
 
         :returns: Now with more "s"
         :raise Exception: Raised occasionally
         :raises ZeroDivisionError: You know why and when
+
+        Some more tests below:
+
+        :rtype: int
+        :vartype x: int
+        :type z: str
+
+        And now for some other stuff
+
+        .. admonition:: TODO
+
+           Create something.
+
+        .. admonition:: Example
+
+           Image shows something.
+
+           .. image:: https://www.debian.org/logos/openlogo-nd-100.png
+
+           .. note::
+              Can only nest admonitions two levels.
+
+        .. image:: https://www.debian.org/logos/openlogo-nd-100.png
+
+        Now you know.
+
+        .. warning::
+
+            Some warning
+            lines.
+
+        * Describe some func in a list
+          across multiple lines:
+
+            .. admonition:: Deprecated since 3.1
+
+                Use `spam` instead.
+
+            .. admonition:: Added in version 2.5
+
+                The *spam* parameter.
+
+        .. caution::
+            Don't touch this!
         """
 
     def reST_directives(self):
diff --git a/setup.py b/setup.py
index a31a3090..86270f3f 100644
--- a/setup.py
+++ b/setup.py
@@ -47,6 +47,7 @@ def _discover_tests():
         provides=["pdoc"],
         obsoletes=["pdoc"],
         install_requires=[
+            "docutils >= 0.16",
             "mako",
             "markdown >= 3.0",
         ],

From 2abd24613a1e0969042d02386c9f7a3354fd3f70 Mon Sep 17 00:00:00 2001
From: muecelee <ialreadygetenough@spam.com>
Date: Wed, 21 Oct 2020 21:01:58 +0200
Subject: [PATCH 3/6] Clean up code wrt linter

---
 pdoc/html_helpers.py  | 26 +++++++++++++++-----------
 pdoc/test/__init__.py |  8 +++++++-
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/pdoc/html_helpers.py b/pdoc/html_helpers.py
index a68d0af1..1f36ee04 100644
--- a/pdoc/html_helpers.py
+++ b/pdoc/html_helpers.py
@@ -15,7 +15,7 @@
 import docutils.nodes
 import docutils.core
 import collections
-from typing import *
+from typing import Union
 
 
 import markdown
@@ -284,7 +284,7 @@ def _reST_string_to_html(text: str) -> str:
 
     @staticmethod
     def _reST_node_to_html(node: docutils.nodes.Node,
-                          doctree: docutils.nodes.document) -> str:
+                           doctree: docutils.nodes.document) -> str:
         """Not all nodes in the doctree provide their reST source or at least the
         starting line in the reST source. This method simply copies the document
         tree and removes all but the node to then publish it.
@@ -346,7 +346,8 @@ def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
 
             # Fill the sections
             try:
-                section = [section for tags, section in tags_to_section_map.items() if tag in tags][0]
+                section = [section for tags, section in tags_to_section_map.items()
+                           if tag in tags][0]
             except IndexError:  # Field is not corresponding to a predefined section like Args
                 section = None
 
@@ -389,13 +390,15 @@ def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
                 markdown.append(f'{section}:\n-----=')
 
                 for field in fields:
-                    field['body'] = field['body'].replace('\n', '\n  ')  # Allow for proper indentation
+                    field['body'] = field['body'].replace('\n', '\n  ')  # For proper indentation
                     if field['name'] or field['type']:
-                        markdown.append(_ToMarkdown._deflist(*_ToMarkdown._fix_indent(field['name'],
-                                                                                      field['type'],
-                                                                                      field['body'])))
-                    else:  # For fields with no name or type (e.g. Returns without type specification)
-                        text = _ToMarkdown._fix_indent(field['name'], field['type'], field['body'])[2]
+                        markdown.append(
+                            _ToMarkdown._deflist(*_ToMarkdown._fix_indent(field['name'],
+                                                                          field['type'],
+                                                                          field['body'])))
+                    else:  # For fields with no name or type (e.g. Returns without type spec)
+                        text = _ToMarkdown._fix_indent(
+                            field['name'], field['type'], field['body'])[2]
                         markdown.append(f':   {text}')
 
         return '\n'.join(markdown)
@@ -600,8 +603,9 @@ def to_markdown(text: str, *,
             reST_regex = fr'^:(?:{"|".join(reST_tags)}).*?:'
             found_reST_tags = re.findall(reST_regex, text, re.MULTILINE)
 
-            # Assume reST-style docstring if any of the above specified tags is present at the beginning of a line.
-            # Could make this more robust, e.g., by checking against the amount of found google or numpy tags
+            # Assume reST-style docstring if any of the above specified tags is present at the
+            # beginning of a line.  Could make this more robust, e.g., by checking against the
+            # amount of found google or numpy tags
             docformat = 'restructuredtext ' if len(found_reST_tags) > 0 else 'numpy,google '
 
         docformat, *_ = docformat.lower().split()
diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
index 4bd72343..2cb615cc 100644
--- a/pdoc/test/__init__.py
+++ b/pdoc/test/__init__.py
@@ -233,6 +233,12 @@ def test_html_ref_links(self):
             )
 
     def test_docformat(self):
+        def test_docformat(self):
+            with self.assertWarns(UserWarning) as cm, \
+                    run_html(EXAMPLE_MODULE, config='docformat="epytext"'):
+                self._basic_html_assertions()
+            self.assertIn('numpy', cm.warning.args[0])
+
         with self.assertWarns(UserWarning) as cm,\
                 run_html(EXAMPLE_MODULE, config='docformat="epytext"'):
             self._basic_html_assertions()
@@ -1544,7 +1550,7 @@ def test_reST(self):
 <div class="admonition caution">
 <p class="admonition-title">Caution!</p>
 <p>Don't touch this!</p>
-</div>'''
+</div>'''  # noqa: 501
         text = inspect.getdoc(self._docmodule.reST)
         html = to_html(text, module=self._module, link=self._link)
 

From 662b1446e3d240aed75dbd0d47676dca9b0401a7 Mon Sep 17 00:00:00 2001
From: muecelee <ialreadygetenough@spam.com>
Date: Wed, 21 Oct 2020 17:52:27 +0200
Subject: [PATCH 4/6] Change reST docstring processing to use docutils

Change "restructuredtext" to "epytext" in test_docformat() since
restructuredtext is now (somewhat) supported
Add trailing spaces to reST tags at docstring type inference

Clean up code wrt linter
---
 pdoc/html_helpers.py              | 222 +++++++++++++++++++++---------
 pdoc/test/__init__.py             | 154 ++++++++++++++++-----
 pdoc/test/example_pkg/__init__.py |  66 ++++++++-
 setup.py                          |   1 +
 4 files changed, 345 insertions(+), 98 deletions(-)

diff --git a/pdoc/html_helpers.py b/pdoc/html_helpers.py
index 23de8085..1f36ee04 100644
--- a/pdoc/html_helpers.py
+++ b/pdoc/html_helpers.py
@@ -12,6 +12,11 @@
 from typing import Callable, Match
 from warnings import warn
 import xml.etree.ElementTree as etree
+import docutils.nodes
+import docutils.core
+import collections
+from typing import Union
+
 
 import markdown
 from markdown.inlinepatterns import InlineProcessor
@@ -266,69 +271,154 @@ def googledoc_sections(match):
         return text
 
     @staticmethod
-    def reST(text: str) -> str:
-        """
-        Convert `text` in reST-style docstring format to Markdown
-        to be further converted later.
+    def _reST_string_to_html(text: str) -> str:
+        """Convert reST text to html using docutils
+
+        :param text: The text to convert
+        :returns: The generated html
         """
-        def reST_sections(match) -> str:
-            # E.g. for ":param arg1: Text" tag is "param", name is "arg1", and body is "Text"
-            tag, name, body = match.groups('')
-            body = textwrap.dedent(body)
+        html = docutils.core.publish_parts(text, writer_name='html')['html_body']
 
-            type_ = None
-            nonlocal active_section
-            active_section_changed = False
-
-            if tag in ['type', 'rtype']:
-                return ''
-            elif tag in ('param', 'parameter', 'arg', 'argument', 'key', 'keyword'):
-                type_ = parameter_types.get(name, None)
-
-                if active_section != 'Args':
-                    active_section = 'Args'
-                    active_section_changed = True
-            elif tag in ('return', 'returns'):
-                if len(return_types) > 0:
-                    type_ = return_types.pop()
-
-                if active_section != 'Returns':
-                    active_section = 'Returns'
-                    active_section_changed = True
-            elif tag in ('raise', 'raises'):
-                if active_section != 'Raises':
-                    active_section = 'Raises'
-                    active_section_changed = True
-
-            if name or type_:
-                text = _ToMarkdown._deflist(*_ToMarkdown._fix_indent(name, type_, body))
-            else:
-                _, _, body = _ToMarkdown._fix_indent(name, type_, body)
-                text = f':   {body}'
+        # Remove the document tag and return
+        return html[23:-8]
 
-            if active_section_changed:
-                text = f'\n{active_section}:\n-----=\n{text}'
-            else:
-                text = f'\n{text}'
+    @staticmethod
+    def _reST_node_to_html(node: docutils.nodes.Node,
+                           doctree: docutils.nodes.document) -> str:
+        """Not all nodes in the doctree provide their reST source or at least the
+        starting line in the reST source. This method simply copies the document
+        tree and removes all but the node to then publish it.
+
+        :node: The node to publish. Must be a child of `doctree` itself
+        :doctree: The document having `node` as a child node
+        :return: The generated html for this node
+        """
+        # Remove all but the given node from the doctree
+        children_copy = doctree.children
+        doctree.children = [doctree.children[doctree.index(node)]]
+
+        # Generate the html for this node/doctree
+        html = docutils.core.publish_from_doctree(doctree, writer_name='html5').decode('utf-8')
 
-            return text
+        # Restore the doctree
+        doctree.children = children_copy
 
-        regex = re.compile(r'^:(\S+)(?:\s(\S+?))?:((?:\n?(?: .*|$))+)', re.MULTILINE)
+        # Return only the relevant part of the html
+        return re.search(r"<div class=\"document\">(.+)<\/div>", html, re.DOTALL).group(1).strip()
 
-        # Get all parameter and return types beforehand, to then use them when substituting
-        # the sections
-        parameter_types = {}
-        return_types = []
-        for tag, name, body in regex.findall(text):
-            if tag == 'type':
-                parameter_types[name] = body.strip()
+    @staticmethod
+    def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
+                                                       docutils.nodes.docinfo]) -> str:
+        """Processes a docutils field list and converts it to markdown.
+        Args, Vars, Returns, and Raises sections are predefined, other sections
+        will be created corresponding to their field names.
+
+        :param field_list: A docutils field list to convert. Can also be a docinfo
+            in case, e.g., only :returns: is specified without any summary text
+        :returns: The generated Markdown. However, it is not pure Markdown, as
+            the field descriptions have been processed with `docutils.process_string` - they
+            therefore are already converted to HTML
+        """
+        # Sort the field list so that types come last  - in case someone defines first the type then
+        # the parameter
+        field_list.children.sort(key=lambda field: 'type' in field[0].rawsource.split()[0])
+
+        # Predefined sections for the generated markdown
+        tags_to_section_map = {
+            ('param', 'parameter', 'arg', 'argument', 'key', 'keyword'): 'Args',
+            ('var', 'ivar', 'cvar'): 'Vars',
+            ('return', 'returns'): 'Returns',
+            ('raise', 'raises'): 'Raises'
+        }
+        sections = collections.OrderedDict([('Args', []), ('Vars', []),
+                                            ('Returns', []), ('Raises', [])])
+
+        # Process the fields
+        for field in field_list:
+            field_name = field.children[0]
+            field_body = field.children[1]
+
+            # Split the field name into its components
+            split = field_name.rawsource.split()
+            tag = split[0]
+            type_ = split[1] if len(split) == 3 else None
+            name = split[2] if len(split) == 3 else split[1] if len(split) == 2 else None
+
+            # Fill the sections
+            try:
+                section = [section for tags, section in tags_to_section_map.items()
+                           if tag in tags][0]
+            except IndexError:  # Field is not corresponding to a predefined section like Args
+                section = None
+
+            if section is not None:
+                sections[section].append({'name': name,
+                                          'type': type_,
+                                          'body': _ToMarkdown._reST_string_to_html(
+                                              field_body.rawsource)})
             elif tag == 'rtype':
-                return_types.append(body.strip())
+                # Set the return type. Assumes that at most one :return: has been specified
+                try:
+                    sections['Returns'][0]['type'] = field_body.rawsource
+                except IndexError:  # Only return type is specified
+                    sections['Returns'].append({'name': None,
+                                                'type': field_body.rawsource.strip(),
+                                                'body': ''})
+            elif 'type' in tag:
+                section = 'Vars' if tag == 'vartype' else 'Args'
+                try:
+                    param_or_var = [x for x in sections[section] if x['name'] == name][0]
+                    param_or_var['type'] = field_body.rawsource.strip()
+                except IndexError:  # Only parameter (or variable) type is specified
+                    sections[section].append({'name': name,
+                                              'type': field_body.rawsource.strip(),
+                                              'body': ''})
+            elif tag == 'meta':
+                pass  # Meta fields should be excluded from the final output
+            else:
+                # Generate sections for tags not covered yet
+                section = sections.get(tag, [])
+                section.append({'name': name,
+                                'type': type_,
+                                'body': _ToMarkdown._reST_string_to_html(field_body.rawsource)})
+                sections[tag] = section
+
+        # Generate the markdown for this field list
+        markdown = []
+        for section, fields in sections.items():
+            if len(fields) > 0:  # Skip empty sections
+                markdown.append(f'{section}:\n-----=')
+
+                for field in fields:
+                    field['body'] = field['body'].replace('\n', '\n  ')  # For proper indentation
+                    if field['name'] or field['type']:
+                        markdown.append(
+                            _ToMarkdown._deflist(*_ToMarkdown._fix_indent(field['name'],
+                                                                          field['type'],
+                                                                          field['body'])))
+                    else:  # For fields with no name or type (e.g. Returns without type spec)
+                        text = _ToMarkdown._fix_indent(
+                            field['name'], field['type'], field['body'])[2]
+                        markdown.append(f':   {text}')
+
+        return '\n'.join(markdown)
 
-        active_section = None  # Keep track of the currently active section (e.g. Args, Returns)
-        text = regex.sub(reST_sections, text)
+    @staticmethod
+    def reST(text: str) -> str:
+        """
+        Convert `text` in reST-style docstring format to Markdown - with embedded html
+        for paragraphs and field descriptions - to be further converted later.
+        """
+        doctree = docutils.core.publish_doctree(text)
 
-        return text
+        generated_markdown = []
+        for section in doctree:
+            if section.tagname in ('field_list', 'docinfo'):
+                generated_markdown.append(_ToMarkdown._reST_field_list_to_markdown(section))
+            else:
+                generated_markdown.append(_ToMarkdown._reST_node_to_html(section, doctree))
+
+        return '\n'.join(generated_markdown)
 
     @staticmethod
     def _admonition(match, module=None, limit_types=None):
@@ -509,41 +599,41 @@ def to_markdown(text: str, *,
 
         # Infer docformat if it hasn't been specified
         if docformat == '':
-            reST_tags = ['param', 'arg', 'type', 'raise', 'except', 'return', 'rtype']
+            reST_tags = ['param ', 'arg ', 'type ', 'raise ', 'except ', 'return', 'rtype']
             reST_regex = fr'^:(?:{"|".join(reST_tags)}).*?:'
             found_reST_tags = re.findall(reST_regex, text, re.MULTILINE)
 
-            # Assume reST-style docstring if any of the above specified tags is present at the beginning of a line.
-            # Could make this more robust, e.g., by checking against the amount of found google or numpy tags
-            if len(found_reST_tags) > 0:
-                docformat = 'reST '
-            else:
-                docformat = 'numpy,google '
+            # Assume reST-style docstring if any of the above specified tags is present at the
+            # beginning of a line.  Could make this more robust, e.g., by checking against the
+            # amount of found google or numpy tags
+            docformat = 'restructuredtext ' if len(found_reST_tags) > 0 else 'numpy,google '
 
         docformat, *_ = docformat.lower().split()
 
-    if not (set(docformat.split(',')) & {'', 'numpy', 'google', 'rest'}):
+    if not (set(docformat.split(',')) & {'', 'numpy', 'google', 'restructuredtext'}):
         warn('__docformat__ value {!r} in module {!r} not supported. '
-             'Supported values are: numpy, google, reST.'.format(docformat, module))
+             'Supported values are: numpy, google, restructuredtext.'.format(docformat, module))
         docformat = 'numpy,google'
 
     with _fenced_code_blocks_hidden(text) as result:
         text = result[0]
 
-        text = _ToMarkdown.admonitions(text, module)
+        if 'restructuredtext' not in docformat:  # Will be handled by docutils
+            text = _ToMarkdown.admonitions(text, module)
 
         if 'google' in docformat:
             text = _ToMarkdown.google(text)
 
         text = _ToMarkdown.doctests(text)
-        text = _ToMarkdown.raw_urls(text)
+        if 'restructuredtext' not in docformat:  # Will be handled by docutils
+            text = _ToMarkdown.raw_urls(text)
 
         # If doing both, do numpy after google, otherwise google-style's
         # headings are incorrectly interpreted as numpy params
         if 'numpy' in docformat:
             text = _ToMarkdown.numpy(text)
 
-        if 'rest' in docformat:
+        if 'restructuredtext' in docformat:
             text = _ToMarkdown.reST(text)
 
         if module and link:
diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
index 946f9f34..2cb615cc 100644
--- a/pdoc/test/__init__.py
+++ b/pdoc/test/__init__.py
@@ -233,8 +233,14 @@ def test_html_ref_links(self):
             )
 
     def test_docformat(self):
+        def test_docformat(self):
+            with self.assertWarns(UserWarning) as cm, \
+                    run_html(EXAMPLE_MODULE, config='docformat="epytext"'):
+                self._basic_html_assertions()
+            self.assertIn('numpy', cm.warning.args[0])
+
         with self.assertWarns(UserWarning) as cm,\
-                run_html(EXAMPLE_MODULE, config='docformat="restructuredtext"'):
+                run_html(EXAMPLE_MODULE, config='docformat="epytext"'):
             self._basic_html_assertions()
         self.assertIn('numpy', cm.warning.args[0])
 
@@ -1403,36 +1409,6 @@ def test_google(self):
         html = to_html(text, module=self._module, link=self._link)
         self.assertEqual(html, expected)
 
-    def test_reST(self):
-        expected = '''<p>Summary line.</p>
-<h2 id="args">Args:</h2>
-<dl>
-<dt><strong><code>arg1</code></strong> :&ensp;<code>int</code></dt>
-<dd>Text1</dd>
-<dt><strong><code>arg2</code></strong> :&ensp;<code>Optional[List[Tuple[str]]]</code></dt>
-<dd>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
- diam nonumy eirmod tempor invidunt</dd>
-<dt><strong><code>arg_arg_3</code></strong> :&ensp;<code>Dict[int, Dict[str, Any]]</code></dt>
-<dd>Y:=H^T<em>X!@#$%^&amp;&amp;</em>()_[]{}';'::</dd>
-</dl>
-<h2 id="returns">Returns:</h2>
-<dl>
-<dt><code>bool</code></dt>
-<dd>True. Or False. Depends</dd>
-<dd>Now with more "s"</dd>
-</dl>
-<h2 id="raises">Raises:</h2>
-<dl>
-<dt><strong><code>Exception</code></strong></dt>
-<dd>Raised occasionally</dd>
-<dt><strong><code>ZeroDivisionError</code></strong></dt>
-<dd>You know why and when</dd>
-</dl>'''
-        text = inspect.getdoc(self._docmodule.reST)
-        html = to_html(text, module=self._module, link=self._link)
-
-        self.assertEqual(html, expected)
-
     def test_doctests(self):
         expected = '''<p>Need an intro paragrapgh.</p>
 <pre><code>&gt;&gt;&gt; Then code is indented one level
@@ -1465,6 +1441,121 @@ def test_doctests(self):
         html = to_html(text, module=self._module, link=self._link)
         self.assertEqual(html, expected)
 
+    def test_reST(self):
+        expected = '''<p>Summary line.</p>
+<p>Some stuff to test like <a class="reference external" href="http://www.python.org">http://www.python.org</a> or <a class="reference external" href="http://www.python.org">link_text</a>.
+Also <em>italic</em> and <strong>bold</strong>. And lists:</p>
+<ul class="simple">
+<li><p>1</p></li>
+<li><p>2</p></li>
+</ul>
+<ol class="arabic simple">
+<li><p>Item</p></li>
+<li><p>Item</p></li>
+</ol>
+<h2 id="args">Args:</h2>
+<dl>
+<dt><strong><code>arg1</code></strong> :&ensp;<code>int</code></dt>
+<dd><p>Text1</p></dd>
+<dt><strong><code>arg2</code></strong> :&ensp;<code>Optional[List[Tuple[str]]]</code></dt>
+<dd><p>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
+diam nonumy eirmod tempor invidunt</p></dd>
+<dt><strong><code>arg_arg_3</code></strong> :&ensp;<code>Dict[int, Dict[str, Any]]</code></dt>
+<dd><p>Y:=H^T<em>X!&#64;#$%^&amp;&amp;</em>()_[]{}';'::{[( :param just_in_case:</p></dd>
+<dt><strong><code>another_parameter</code></strong> :&ensp;<code>str</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<h2 id="vars">Vars:</h2>
+<dl>
+<dt><strong><code>x</code></strong></dt>
+<dd><p>Description of variable x</p></dd>
+<dt><strong><code>y</code></strong> :&ensp;<code>List[bool]</code></dt>
+<dd><p>Description of variable y</p></dd>
+<dt><strong><code>z</code></strong> :&ensp;<code>str</code></dt>
+<dd><p>Descriptions can also be placed in a new line.</p>
+<p>And span multiple lines.</p></dd>
+</dl>
+<h2 id="returns">Returns:</h2>
+<dl>
+<dt><code>bool</code></dt>
+<dd><p>True. Or False. Depends</p></dd>
+</dl>
+<p>A paragraph to split the field list into two.</p>
+<h2 id="returns_1">Returns:</h2>
+<dl>
+<dd>
+<dl>
+<dt><p>Now with more &quot;s&quot;</p></dt>
+<dt>Raises:</dt>
+<dt>-----=</dt>
+<dt><strong><code>Exception</code></strong></dt>
+<dd><p>Raised occasionally</p></dd>
+</dl>
+</dd>
+<dt><strong><code>ZeroDivisionError</code></strong></dt>
+<dd><p>You know why and when</p></dd>
+</dl>
+<p>Some more tests below:</p>
+<h2 id="args_1">Args:</h2>
+<dl>
+<dt><strong><code>z</code></strong> :&ensp;<code>str</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<h2 id="vars_1">Vars:</h2>
+<dl>
+<dt><strong><code>x</code></strong> :&ensp;<code>int</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<h2 id="returns_2">Returns:</h2>
+<dl>
+<dt><code>int</code></dt>
+<dd>&nbsp;</dd>
+</dl>
+<p>And now for some other stuff</p>
+<div class="admonition admonition-todo">
+<p class="admonition-title">TODO</p>
+<p>Create something.</p>
+</div>
+<div class="admonition admonition-example">
+<p class="admonition-title">Example</p>
+<p>Image shows something.</p>
+<img alt="https://www.debian.org/logos/openlogo-nd-100.png" src="https://www.debian.org/logos/openlogo-nd-100.png" />
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>Can only nest admonitions two levels.</p>
+</div>
+</div>
+<p><img alt="https://www.debian.org/logos/openlogo-nd-100.png" src="https://www.debian.org/logos/openlogo-nd-100.png" /></p>
+<p>Now you know.</p>
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>Some warning
+lines.</p>
+</div>
+<ul>
+<li><p>Describe some func in a list
+across multiple lines:</p>
+<blockquote>
+<div class="admonition admonition-deprecated-since-3-1">
+<p class="admonition-title">Deprecated since 3.1</p>
+<p>Use <cite>spam</cite> instead.</p>
+</div>
+<div class="admonition admonition-added-in-version-2-5">
+<p class="admonition-title">Added in version 2.5</p>
+<p>The <em>spam</em> parameter.</p>
+</div>
+</blockquote>
+</li>
+</ul>
+<div class="admonition caution">
+<p class="admonition-title">Caution!</p>
+<p>Don't touch this!</p>
+</div>'''  # noqa: 501
+        text = inspect.getdoc(self._docmodule.reST)
+        html = to_html(text, module=self._module, link=self._link)
+
+        self.assertEqual(html, expected)
+
     def test_reST_directives(self):
         expected = '''<div class="admonition todo">
 <p class="admonition-title">TODO</p>
@@ -1506,6 +1597,7 @@ def test_reST_directives(self):
 </div>'''
         text = inspect.getdoc(self._docmodule.reST_directives)
         html = to_html(text, module=self._module, link=self._link)
+
         self.assertEqual(html, expected)
 
     def test_reST_include(self):
diff --git a/pdoc/test/example_pkg/__init__.py b/pdoc/test/example_pkg/__init__.py
index 559cbb0c..1176bfb4 100644
--- a/pdoc/test/example_pkg/__init__.py
+++ b/pdoc/test/example_pkg/__init__.py
@@ -302,19 +302,83 @@ def reST(self):
         """
         Summary line.
 
+        Some stuff to test like http://www.python.org or `link_text <http://www.python.org>`_.
+        Also *italic* and **bold**. And lists:
+
+        * 1
+        * 2
+
+        #. Item
+        #. Item
+
         :param arg1: Text1
         :parameter arg2: Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
             diam nonumy eirmod tempor invidunt
-        :arg arg_arg_3: Y:=H^T*X!@#$%^&&*()_[]{}';'::
+        :arg arg_arg_3: Y:=H^T*X!@#$%^&&*()_[]{}';'::{[( :param just_in_case:
+        :key str another_parameter:
         :type arg1: int
         :type arg2: Optional[List[Tuple[str]]]
         :type arg_arg_3: Dict[int, Dict[str, Any]]
+        :var x: Description of variable x
+        :ivar y: Description of variable y
+        :cvar str z:
+            Descriptions can also be placed in a new line.
+
+            And span multiple lines.
+        :vartype y: List[bool]
         :return: True. Or False. Depends
         :rtype: bool
+        :meta: This should not be displayed in the generated output document
+
+        A paragraph to split the field list into two.
 
         :returns: Now with more "s"
         :raise Exception: Raised occasionally
         :raises ZeroDivisionError: You know why and when
+
+        Some more tests below:
+
+        :rtype: int
+        :vartype x: int
+        :type z: str
+
+        And now for some other stuff
+
+        .. admonition:: TODO
+
+           Create something.
+
+        .. admonition:: Example
+
+           Image shows something.
+
+           .. image:: https://www.debian.org/logos/openlogo-nd-100.png
+
+           .. note::
+              Can only nest admonitions two levels.
+
+        .. image:: https://www.debian.org/logos/openlogo-nd-100.png
+
+        Now you know.
+
+        .. warning::
+
+            Some warning
+            lines.
+
+        * Describe some func in a list
+          across multiple lines:
+
+            .. admonition:: Deprecated since 3.1
+
+                Use `spam` instead.
+
+            .. admonition:: Added in version 2.5
+
+                The *spam* parameter.
+
+        .. caution::
+            Don't touch this!
         """
 
     def reST_directives(self):
diff --git a/setup.py b/setup.py
index a31a3090..86270f3f 100644
--- a/setup.py
+++ b/setup.py
@@ -47,6 +47,7 @@ def _discover_tests():
         provides=["pdoc"],
         obsoletes=["pdoc"],
         install_requires=[
+            "docutils >= 0.16",
             "mako",
             "markdown >= 3.0",
         ],

From 925a938d72a281c18c5ecb31b248b4399877bf22 Mon Sep 17 00:00:00 2001
From: muecelee <ialreadygetenough@spam.com>
Date: Wed, 21 Oct 2020 21:12:53 +0200
Subject: [PATCH 5/6] Remove accidental code duplication in test_docformat

---
 pdoc/test/__init__.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/pdoc/test/__init__.py b/pdoc/test/__init__.py
index 2cb615cc..724a593c 100644
--- a/pdoc/test/__init__.py
+++ b/pdoc/test/__init__.py
@@ -233,12 +233,6 @@ def test_html_ref_links(self):
             )
 
     def test_docformat(self):
-        def test_docformat(self):
-            with self.assertWarns(UserWarning) as cm, \
-                    run_html(EXAMPLE_MODULE, config='docformat="epytext"'):
-                self._basic_html_assertions()
-            self.assertIn('numpy', cm.warning.args[0])
-
         with self.assertWarns(UserWarning) as cm,\
                 run_html(EXAMPLE_MODULE, config='docformat="epytext"'):
             self._basic_html_assertions()

From 6d35ccad383b43fe1f4f324594e73a7a4799cf17 Mon Sep 17 00:00:00 2001
From: muecelee <ialreadygetenough@spam.com>
Date: Fri, 23 Oct 2020 12:49:49 +0200
Subject: [PATCH 6/6] Fix mypy issues

In html_helpers.reST_node_to_html() return an empty string in case the
regex can't find a div with class "document"
Specifiy variable types where not inferable
---
 pdoc/html_helpers.py | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/pdoc/html_helpers.py b/pdoc/html_helpers.py
index 1f36ee04..1334e1fd 100644
--- a/pdoc/html_helpers.py
+++ b/pdoc/html_helpers.py
@@ -14,8 +14,8 @@
 import xml.etree.ElementTree as etree
 import docutils.nodes
 import docutils.core
-import collections
-from typing import Union
+from collections import OrderedDict
+from typing import Union, List, Dict, Optional
 
 
 import markdown
@@ -304,7 +304,15 @@ def _reST_node_to_html(node: docutils.nodes.Node,
         doctree.children = children_copy
 
         # Return only the relevant part of the html
-        return re.search(r"<div class=\"document\">(.+)<\/div>", html, re.DOTALL).group(1).strip()
+        match = re.search(r'<div class="document">(.+)</div>', html, re.DOTALL)
+
+        if match is not None:
+            return match.group(1).strip()
+        else:
+            # The generated HTML from docutils.publish_from_doctree() should always contain a
+            # div with class "document" in which all generated content is located. However, in case
+            # it doesn't, it's probably empty so return an empty string
+            return ''
 
     @staticmethod
     def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
@@ -330,8 +338,8 @@ def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
             ('return', 'returns'): 'Returns',
             ('raise', 'raises'): 'Raises'
         }
-        sections = collections.OrderedDict([('Args', []), ('Vars', []),
-                                            ('Returns', []), ('Raises', [])])
+        sections: OrderedDict[str, List[Dict[str, Optional[str]]]] = OrderedDict(
+            [('Args', []), ('Vars', []), ('Returns', []), ('Raises', [])])
 
         # Process the fields
         for field in field_list:
@@ -346,8 +354,8 @@ def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
 
             # Fill the sections
             try:
-                section = [section for tags, section in tags_to_section_map.items()
-                           if tag in tags][0]
+                section: Optional[str] = [section for tags, section in tags_to_section_map.items()
+                                          if tag in tags][0]
             except IndexError:  # Field is not corresponding to a predefined section like Args
                 section = None
 
@@ -377,11 +385,11 @@ def _reST_field_list_to_markdown(field_list: Union[docutils.nodes.field_list,
                 pass  # Meta fields should be excluded from the final output
             else:
                 # Generate sections for tags not covered yet
-                section = sections.get(tag, [])
-                section.append({'name': name,
-                                'type': type_,
-                                'body': _ToMarkdown._reST_string_to_html(field_body.rawsource)})
-                sections[tag] = section
+                new_section = sections.get(tag, [])
+                new_section.append({'name': name,
+                                    'type': type_,
+                                    'body': _ToMarkdown._reST_string_to_html(field_body.rawsource)})
+                sections[tag] = new_section
 
         # Generate the markdown for this field list
         markdown = []