update sphinx==3.0

docs/conf.py

@@ -15,6 +15,33 @@
 import sys, os, re
 import mock
 import inspect
+from sphinx.domains import Domain
+
+class GithubURLDomain(Domain):
+    """
+    Resolve certain links in markdown files to github source.
+    """
+
+    name = "githuburl"
+    ROOT = "https://github.com/tensorpack/tensorpack/blob/master/"
+
+    def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode):
+        github_url = None
+        if ".html" not in target:
+            if target.startswith("../../") and not target.startswith("../../modules"):
+                url = target.replace("../", "")
+                github_url = url
+
+        if github_url is not None:
+            if github_url.endswith("README"):
+                # bug of recommonmark.
+                # https://github.com/readthedocs/recommonmark/blob/ddd56e7717e9745f11300059e4268e204138a6b1/recommonmark/parser.py#L152-L155
+                github_url += ".md"
+            print(f"Ref {target} resolved to github:{github_url}")
+            contnode["refuri"] = self.ROOT + github_url
+            return [("githuburl:any", contnode)]
+        else:
+            return []
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -43,6 +70,9 @@
     MOCK_MODULES.extend(['tensorflow.python.training.monitored_session'])
     MOCK_MODULES.extend(['tensorflow.python.training'])
     MOCK_MODULES.extend(['tensorflow.python.client'])
+    MOCK_MODULES.extend(['tensorflow.python.framework'])
+    MOCK_MODULES.extend(['tensorflow.python.platform'])
+    MOCK_MODULES.extend(['tensorflow.python.tools'])
     MOCK_MODULES.extend(['tensorflow.contrib.graph_editor'])
 
 for mod_name in MOCK_MODULES:
@@ -55,12 +85,13 @@
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-needs_sphinx = '1.4'
+needs_sphinx = '3.0'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'recommonmark',
     'sphinx.ext.autodoc',
     'sphinx.ext.todo',
     'sphinx.ext.napoleon',
@@ -92,11 +123,6 @@
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
-# to support markdown
-from recommonmark.parser import CommonMarkParser
-source_parsers = {
-    '.md': CommonMarkParser,
-}
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 source_suffix = ['.rst', '.md']
@@ -109,7 +135,7 @@
 
 # General information about the project.
 project = u'tensorpack'
-copyright = u'2015 - 2019, Yuxin Wu, et al.'
+copyright = u'2015 - 2020, Yuxin Wu, et al.'
 author = u'Yuxin Wu, et al.'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -430,23 +456,14 @@ def autodoc_skip_member(app, what, name, obj, skip, options):
                 return True
     return None
 
-def url_resolver(url):
-    if '.html' not in url:
-        return "https://github.com/tensorpack/tensorpack/blob/master/" + url
-    else:
-        if ON_RTD:
-            return "http://tensorpack.readthedocs.io/" + url
-        else:
-            return '/' + url
-
 def setup(app):
     from recommonmark.transform import AutoStructify
+    app.add_domain(GithubURLDomain)
     app.connect('autodoc-process-signature', process_signature)
     app.connect('autodoc-skip-member', autodoc_skip_member)
     app.add_config_value(
         'recommonmark_config',
-        {'url_resolver': url_resolver,
-         'auto_toc_tree_section': 'Contents',
+        {'auto_toc_tree_section': 'Contents',
          'enable_math': True,
          'enable_inline_math': True,
          'enable_eval_rst': True

docs/requirements.txt

@@ -1,9 +1,9 @@
 termcolor
 numpy
 tqdm
-docutils>=0.14
-Sphinx>=1.6
-recommonmark==0.4.0
+docutils==0.16
+Sphinx==3.0.0
+recommonmark==0.6.0
 sphinx_rtd_theme
 mock
 matplotlib

docs/tutorial/callback.md

@@ -80,7 +80,7 @@ These features are not always necessary, but think about how messy the main loop
 were to write these logic together with the loops, and how easy your life will be if you could enable
 these features with just one line when you need them.
 
-See [list of callbacks](../modules/callbacks.html)
+See [list of callbacks](../modules/callbacks)
 for a long list of tensorpack builtin callbacks.
-See [Write a callback](extend/callback.html)
+See [Write a callback](extend/callback.md)
 for details on how callbacks work, what they can do, and how to write them.

docs/tutorial/dataflow.md

@@ -21,14 +21,14 @@ You can simply use DataFlow as a data processing pipeline and plug it into your
 ### Load Raw Data
 We do not make any assumptions about your data format.
 You would usually want to write the source DataFlow (`MyDataFlow` in the example below) for your own data format.
-See [another tutorial](extend/dataflow.html) for simple instructions on writing a DataFlow.
+See [another tutorial](extend/dataflow.md) for simple instructions on writing a DataFlow.
 
 ### Assemble the Pipeline
 There are a lot of existing DataFlow utilities in tensorpack, which you can use to assemble
 the source DataFlow with complex data pipeline.
-A common pipeline usually would 
-__read from disk (or other sources), 
-apply transformations, 
+A common pipeline usually would
+__read from disk (or other sources),
+apply transformations,
 group into batches, prefetch data__, etc, and all __run in parallel__.
 A simple DataFlow pipeline is like the following:
 
@@ -43,19 +43,19 @@ df = BatchData(df, 128)
 df = MultiProcessRunnerZMQ(df, 3)
 ````
 
-A list of built-in DataFlow to use can be found at [API docs](../modules/dataflow.html).
-You can also find complicated real-life DataFlow pipelines in the [ImageNet training script](../examples/ImageNetModels/imagenet_utils.py)
+A list of built-in DataFlow to use can be found at [API docs](../modules/dataflow).
+You can also find complicated real-life DataFlow pipelines in the [ImageNet training script](../../examples/ImageNetModels/imagenet_utils.py)
 or other tensorpack examples.
 
 ### Parallelize the Pipeline
 
 DataFlow includes **carefully optimized** parallel runners and parallel mappers: `Multi{Thread,Process}{Runner,MapData}`.
 Runners execute multiple clones of a dataflow in parallel.
 Mappers execute a mapping function in parallel on top of an existing dataflow.
-You can find details in the [API docs](../modules/dataflow.html) under the
+You can find details in the [API docs](../modules/dataflow) under the
 "parallel" and "parallel_map" section.
 
-[Parallel DataFlow tutorial](parallel-dataflow.html) gives a deeper dive
+[Parallel DataFlow tutorial](parallel-dataflow.md) gives a deeper dive
 on how to use them to optimize your data pipeline.
 
 ### Run the DataFlow
@@ -77,6 +77,6 @@ for dp in df:
 ### Why DataFlow?
 
 It's **easy and fast**.
-For more discussions, see [Why DataFlow?](/tutorial/philosophy/dataflow.html)
+For more discussions, see [Why DataFlow?](./philosophy/dataflow.md)
 Nevertheless, using DataFlow is not required in tensorpack.
 Tensorpack supports data loading with native TF operators / TF datasets as well.

docs/tutorial/efficient-dataflow.md

@@ -10,31 +10,31 @@ or your own code as well.
 **What we are going to do**: We'll use ILSVRC12 dataset, which contains 1.28 million images.
 The original images (JPEG compressed) are 140G in total.
 The average resolution is about 400x350 <sup>[[1]]</sup>.
-Following the [ResNet example](../examples/ResNet), we need images in their original resolution,
+Following the [ResNet example](../../examples/ResNet), we need images in their original resolution,
 so we will read the original dataset (instead of a down-sampled version), and
 then apply complicated preprocessing to it.
 We hope to reach a speed of **1k~5k images per second**, to keep GPUs busy.
 
 Some things to know before reading:
 
-1. You are recommended to read the [Parallel DataFlow Tutorial](parallel-dataflow.html) first.
+1. You are recommended to read the [Parallel DataFlow Tutorial](./parallel-dataflow.md) first.
 1. You only need the data loader to be **fast enough, but not faster**.
    See [How Fast Do You Actually Need](philosophy/dataflow.html#how-fast-do-you-actually-need) for details.
-   For smaller datasets (e.g. several GBs of images with lightweight preprocessing), 
+   For smaller datasets (e.g. several GBs of images with lightweight preprocessing),
    a simple reader plus some multiprocess runner is usually fast enough.
 1. Having a fast Python generator **alone** may or may not improve your overall training speed.
-	 You need mechanisms to hide the latency of **all** preprocessing stages, as mentioned in the
-	 [InputSource tutorial](extend/input-source.html).
+   You need mechanisms to hide the latency of **all** preprocessing stages, as mentioned in the
+   [InputSource tutorial](./extend/input-source.md).
 1. Reading training set and validation set are different.
-	 In training it's OK to reorder, regroup, or even duplicate some datapoints, as long as the
-	 data distribution stays the same.
-	 But in validation we often need the exact set of data, to be able to compute a correct and comparable score.
-	 This will affect how we build the DataFlow.
+   In training it's OK to reorder, regroup, or even duplicate some datapoints, as long as the
+   data distribution stays the same.
+   But in validation we often need the exact set of data, to be able to compute a correct and comparable score.
+   This will affect how we build the DataFlow.
 1. The actual performance would depend on not only the disk, but also memory (for caching) and CPU (for data processing).
-	You may need to tune the parameters (#processes, #threads, size of buffer, etc.)
-	or change the pipeline for new tasks and new machines to achieve the best performance.
+  You may need to tune the parameters (#processes, #threads, size of buffer, etc.)
+  or change the pipeline for new tasks and new machines to achieve the best performance.
     The solutions in this tutorial may not help you.
-    To improve your own DataFlow, read the 
+    To improve your own DataFlow, read the
     [performance tuning tutorial](performance-tuning.html#investigate-dataflow)
     before performing or asking about any actual optimizations.
 
@@ -62,7 +62,7 @@ for filename, label in np.random.shuffle(filelist):
 ```
 
 And `ds1` batch the datapoints from `ds0`, so that we can measure the speed of this DataFlow in terms of "batches per second".
-By default, `BatchData` should stack the datapoints into an `numpy.ndarray`, 
+By default, `BatchData` should stack the datapoints into an `numpy.ndarray`,
 but since original ImageNet images are of different shapes, we use
 `use_list=True** so that it produces lists for now.
 
@@ -75,45 +75,50 @@ Image decoding in `cv2.imread` may also be a bottleneck at this early stage, sin
 
 We will now add the cheapest pre-processing now to get an ndarray in the end instead of a list
 (because training will need ndarray eventually):
+
 ```eval_rst
+
 .. code-block:: python
-	  :emphasize-lines: 2,3
+    :emphasize-lines: 2,3
+
+    ds = dataset.ILSVRC12('/path/to/ILSVRC12', 'train', shuffle=True)
+    ds = AugmentImageComponent(ds, [imgaug.Resize(224)])
+    ds = BatchData(ds, 256)
 
-		ds = dataset.ILSVRC12('/path/to/ILSVRC12', 'train', shuffle=True)
-		ds = AugmentImageComponent(ds, [imgaug.Resize(224)])
-		ds = BatchData(ds, 256)
 ```
-You'll start to observe slow down after adding more pre-processing (such as those in the [ResNet example](../examples/ImageNetModels/imagenet_utils.py)).
+
+You'll start to observe slow down after adding more pre-processing (such as those in the [ResNet example](../../examples/ImageNetModels/imagenet_utils.py)).
 Now it's time to add threads or processes:
 ```eval_rst
 .. code-block:: python
-	  :emphasize-lines: 3
+    :emphasize-lines: 3
 
-		ds0 = dataset.ILSVRC12('/path/to/ILSVRC12', 'train', shuffle=True)
-		ds1 = AugmentImageComponent(ds0, lots_of_augmentors)
-		ds = MultiProcessRunnerZMQ(ds1, num_proc=25)
-		ds = BatchData(ds, 256)
+    ds0 = dataset.ILSVRC12('/path/to/ILSVRC12', 'train', shuffle=True)
+    ds1 = AugmentImageComponent(ds0, lots_of_augmentors)
+    ds = MultiProcessRunnerZMQ(ds1, num_proc=25)
+    ds = BatchData(ds, 256)
 ```
+
 Here we fork 25 processes to run `ds1`, and collect their output through ZMQ IPC protocol.
 You can also apply parallel runner after batching, of course.
 
 ### Parallel Map
 The above DataFlow might be fast, but since it forks the ImageNet reader (`ds0`),
 it's **not a good idea to use it for validation** (for reasons mentioned at top.
-More details at the [Parallel DataFlow Tutorial](parallel-dataflow) and the [documentation](../modules/dataflow.html#tensorpack.dataflow.MultiProcessRunnerZMQ)).
+More details at the [Parallel DataFlow Tutorial](./parallel-dataflow.md) and the [documentation](../modules/dataflow.html#tensorpack.dataflow.MultiProcessRunnerZMQ)).
 Alternatively, you can use parallel mapper like this:
 
 ```eval_rst
 .. code-block:: python
-	  :emphasize-lines: 3-6
-
-		ds0 = dataset.ILSVRC12('/path/to/ILSVRC12', 'train', shuffle=True)
-		augmentor = AugmentorList(lots_of_augmentors)
-		ds1 = MultiThreadMapData(
-				ds0, num_thread=25,
-				map_func=lambda dp: [augmentor.augment(dp[0]), dp[1]], buffer_size=1000)
-		# ds1 = MultiProcessRunnerZMQ(ds1, num_proc=1)
-		ds = BatchData(ds1, 256)
+    :emphasize-lines: 3-6
+
+    ds0 = dataset.ILSVRC12('/path/to/ILSVRC12', 'train', shuffle=True)
+    augmentor = AugmentorList(lots_of_augmentors)
+    ds1 = MultiThreadMapData(
+        ds0, num_thread=25,
+        map_func=lambda dp: [augmentor.augment(dp[0]), dp[1]], buffer_size=1000)
+    # ds1 = MultiProcessRunnerZMQ(ds1, num_proc=1)
+    ds = BatchData(ds1, 256)
 ```
 `MultiThreadMapData` launches a thread pool to fetch data and apply the mapping function on **a single
 instance of** `ds0`. This is done by an intermediate buffer of size 1000 to hide the mapping latency.
@@ -130,17 +135,17 @@ If you identify this as a bottleneck, you can also use:
 
 ```eval_rst
 .. code-block:: python
-	  :emphasize-lines: 5-6
-
-		ds0 = dataset.ILSVRC12Files('/path/to/ILSVRC12', 'train', shuffle=True)
-		augmentor = AugmentorList(lots_of_augmentors)
-		ds1 = MultiThreadMapData(
-				ds0, num_thread=25,
-				map_func=lambda dp:
-					[augmentor.augment(cv2.imread(dp[0], cv2.IMREAD_COLOR)), dp[1]],
-				buffer_size=1000)
-		ds1 = MultiProcessRunnerZMQ(ds1, num_proc=1)
-		ds = BatchData(ds1, 256)
+    :emphasize-lines: 5-6
+
+    ds0 = dataset.ILSVRC12Files('/path/to/ILSVRC12', 'train', shuffle=True)
+    augmentor = AugmentorList(lots_of_augmentors)
+    ds1 = MultiThreadMapData(
+        ds0, num_thread=25,
+        map_func=lambda dp:
+          [augmentor.augment(cv2.imread(dp[0], cv2.IMREAD_COLOR)), dp[1]],
+        buffer_size=1000)
+    ds1 = MultiProcessRunnerZMQ(ds1, num_proc=1)
+    ds = BatchData(ds1, 256)
 ```
 
 Let's summarize what the above dataflow does:
@@ -190,11 +195,11 @@ As a reference, on Samsung SSD 850, the uncached speed is about 16it/s.
 
 ```eval_rst
 .. code-block:: python
-	  :emphasize-lines: 2
+    :emphasize-lines: 2
 
-	  ds = LMDBSerializer.load('/path/to/ILSVRC-train.lmdb', shuffle=False)
-	  ds = LocallyShuffleData(ds, 50000)
-	  ds = BatchData(ds, 256, use_list=True)
+    ds = LMDBSerializer.load('/path/to/ILSVRC-train.lmdb', shuffle=False)
+    ds = LocallyShuffleData(ds, 50000)
+    ds = BatchData(ds, 256, use_list=True)
 ```
 Instead of shuffling all the training data in every epoch (which would require random read),
 the added line above maintains a buffer of datapoints and shuffle them once a while.
@@ -237,7 +242,7 @@ Let me summarize what this DataFlow does:
 
 1. One process reads LMDB file, shuffle them in a buffer and put them into a ZMQ pipe (used by `MultiProcessMapDataZMQ`).
 2. 25 processes take items from the queue, decode and process them into [image, label] pairs, and
-	 send them through ZMQ IPC pipe to the main process.
+   send them through ZMQ IPC pipe to the main process.
 3. The main process takes data from the pipe, makes batches.
 
 The two DataFlow mentioned in this tutorial (both random read and sequential read) can run at a speed of 1k ~ 5k images per second,
@@ -275,7 +280,7 @@ TestDataSpeed(df).start()
 ## Common Issues on Windows:
 
 1. Windows does not support IPC protocol of ZMQ. You can only use `MultiProcessRunner`,
-   `MultiThreadRunner`, and `MultiThreadMapData`. But you cannot use 
+   `MultiThreadRunner`, and `MultiThreadMapData`. But you cannot use
    `MultiProcessRunnerZMQ` or `MultiProcessMapData` (which is an alias of `MultiProcessMapDataZMQ`).
 2. Windows needs to pickle your dataflow to run it in multiple processes.
    As a result you cannot use lambda functions for mappings, like the examples above.