-
Notifications
You must be signed in to change notification settings - Fork 10
/
conf.py
449 lines (362 loc) · 14.6 KB
/
conf.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import re
import zipfile
from pathlib import Path
from datetime import date
import importlib
import json
import sphinx_rtd_theme
import git
import yaml
TMP_FOLDER = Path(__file__).parent / "tmp"
TMP_FOLDER.mkdir(exist_ok=True)
JSON_COMPATIBILITY_TABLE_FILE = TMP_FOLDER / "releases.json"
with open("additional/releases.yaml") as f:
compat_table = yaml.safe_load(f)
with open(JSON_COMPATIBILITY_TABLE_FILE, "w") as f:
json.dump(compat_table, f)
repo = git.Repo(search_parent_directories=True)
current_commit = repo.head.commit
tagged_commits = [tag.commit for tag in repo.tags]
if os.environ.get("READTHEDOCS_VERSION_TYPE") == "tag" or current_commit in tagged_commits:
# Index 0 means latest release
SUBSTRA_VERSION = compat_table["releases"][0]["components"]["substra"]["version"]
TOOLS_VERSION = compat_table["releases"][0]["components"]["substra-tools"]["version"]
SUBSTRAFL_VERSION = compat_table["releases"][0]["components"]["substrafl"]["version"]
else:
SUBSTRA_VERSION = "main"
TOOLS_VERSION = "main"
SUBSTRAFL_VERSION = "main"
print(
f"Versions of the components used:"
f"\n - substra: {SUBSTRA_VERSION}"
f"\n - substra-tools: {TOOLS_VERSION}"
f"\n - substrafl: {SUBSTRAFL_VERSION}"
)
class SubSectionTitleOrder:
"""Sort example gallery by title of subsection.
Assumes README.txt exists for all subsections and uses the subsection with
dashes, '---', as the adornment.
This class is adapted from sklearn
"""
def __init__(self, src_dir):
self.src_dir = src_dir
self.regex = re.compile(r"^([\w ]+)\n-", re.MULTILINE)
def __repr__(self):
return "<%s>" % (self.__class__.__name__,)
def __call__(self, directory):
src_path = os.path.normpath(os.path.join(self.src_dir, directory))
# Forces Release Highlights to the top
if os.path.basename(src_path) == "release_highlights":
return "0"
readme = os.path.join(src_path, "README.txt")
try:
with open(readme, "r") as f:
content = f.read()
except FileNotFoundError:
return directory
title_match = self.regex.search(content)
if title_match is not None:
return title_match.group(1)
return directory
# Nbsphinx config
nbsphinx_thumbnails = {
"examples/substra_core/diabetes_example/run_diabetes": "_static/example_thumbnail/diabetes.png",
"examples/substra_core/titanic_example/run_titanic": "_static/example_thumbnail/titanic.jpg",
"examples/substrafl/get_started/run_mnist_torch": "_static/example_thumbnail/mnist.png",
"examples/substrafl/go_further/run_diabetes_substrafl": "_static/example_thumbnail/diabetes.png",
"examples/substrafl/go_further/run_iris_sklearn": "_static/example_thumbnail/iris.jpg",
"examples/substrafl/go_further/run_mnist_cyclic": "_static/example_thumbnail/cyclic-mnist.png",
}
nbsphinx_prolog = r"""
{% set docname = 'docs/source/' + env.doc2path(env.docname, base=None) %}
.. raw:: html
<div class="notebook note">
Launch notebook online <span style="white-space: nowrap;"><a href="https://mybinder.org/v2/gh/Substra/substra-documentation/{{ env.config.release|e }}?filepath={{ docname|e }}"><img alt="Binder badge" src="https://mybinder.org/badge_logo.svg" style="vertical-align:text-bottom"></a></span>
or download it <span style="white-space: nowrap;"><a href="{{ env.docname.split('/')|last|e + '.ipynb' }}" download><img alt="Download badge" src="https://img.shields.io/badge/download_-notebook-orange?logo=jupyter" style="vertical-align:text-bottom"></a></span>
</div>
"""
nbsphinx_epilog = nbsphinx_prolog
nbsphinx_execute = "never"
# zip the assets directory found in the examples directory and place it in the current dir
def zip_dir(source_dir, zip_file_name):
# Create archive with compressed files
with zipfile.ZipFile(file=TMP_FOLDER / zip_file_name, mode="w", compression=zipfile.ZIP_DEFLATED) as ziph:
for root, _, files in os.walk(source_dir):
for file in files:
ziph.write(
os.path.join(root, file),
os.path.relpath(os.path.join(root, file), os.path.join(source_dir, "..")),
)
assets_dir_titanic = Path(__file__).parent / "examples" / "substra_core" / "titanic_example" / "assets"
zip_dir(assets_dir_titanic, "titanic_assets.zip")
assets_dir_diabetes = Path(__file__).parent / "examples" / "substra_core" / "diabetes_example" / "assets"
zip_dir(assets_dir_diabetes, "diabetes_assets.zip")
assets_dir_substrafl_torch_fedavg = (
Path(__file__).parent / "examples" / "substrafl" / "get_started" / "torch_fedavg_assets"
)
zip_dir(assets_dir_substrafl_torch_fedavg, "torch_fedavg_assets.zip")
assets_dir_substrafl_diabetes = (
Path(__file__).parent / "examples" / "substrafl" / "go_further" / "diabetes_substrafl_assets"
)
zip_dir(assets_dir_substrafl_diabetes, "diabetes_substrafl_assets.zip")
assets_dir_substrafl_sklearn_fedavg = (
Path(__file__).parent / "examples" / "substrafl" / "go_further" / "sklearn_fedavg_assets"
)
zip_dir(assets_dir_substrafl_sklearn_fedavg, "sklearn_fedavg_assets.zip")
assets_dir_substrafl_sklearn_fedavg = (
Path(__file__).parent / "examples" / "substrafl" / "go_further" / "torch_cyclic_assets"
)
zip_dir(assets_dir_substrafl_sklearn_fedavg, "torch_cyclic_assets.zip")
# Copy the source documentation files from substra and substrafl to their right place
# in the substra-documentation repository
from dataclasses import dataclass
from distutils.dir_util import copy_tree
import subprocess
import shutil
import sys
import typing
EDITABLE_LIB_PATH = Path(__file__).resolve().parents[1] / "src"
@dataclass
class Repo:
pkg_name: str
repo_name: str
installation_cmd: str
version: str
doc_dir: typing.Optional[str] = None
dest_doc_dir: typing.Optional[str] = None
SUBSTRA_REPOS = [
Repo(
pkg_name="substra",
repo_name="substra",
installation_cmd="#egg=substra",
version=SUBSTRA_VERSION,
doc_dir="references",
dest_doc_dir="documentation/references",
),
Repo(
pkg_name="substrafl",
repo_name="substrafl",
installation_cmd="#egg=substrafl[dev]",
version=SUBSTRAFL_VERSION,
doc_dir="docs/api",
dest_doc_dir="substrafl_doc/api",
),
Repo(
pkg_name="substratools",
repo_name="substra-tools",
installation_cmd="#egg=substratools",
version=TOOLS_VERSION,
),
]
def install_dependency(library_name, repo_name, repo_args, version):
try:
subprocess.run(
args=[
sys.executable,
"-m",
"pip",
"install",
"--src",
str(EDITABLE_LIB_PATH),
"--editable",
f"git+https://github.com/substra/{repo_name}.git@{version}{repo_args}",
],
check=True,
capture_output=True,
)
except subprocess.CalledProcessError as e:
print(e.stderr)
print(e.stdout)
raise
importlib.invalidate_caches()
sys.path.insert(0, str(EDITABLE_LIB_PATH / library_name))
def copy_source_files(src, dest):
full_dest_path = Path(__file__).resolve().parent / dest
if full_dest_path.exists():
shutil.rmtree(full_dest_path)
copy_tree(str(src), str(full_dest_path))
for repo in SUBSTRA_REPOS:
install_dependency(
library_name=repo.pkg_name,
repo_name=repo.repo_name,
repo_args=repo.installation_cmd,
version=repo.version,
)
if repo.doc_dir is not None:
imported_module = importlib.import_module(repo.pkg_name)
source_path = Path(imported_module.__file__).resolve().parents[1] / repo.doc_dir
copy_source_files(source_path, repo.dest_doc_dir)
myst_all_links_external = True
# reformat links to a section in a markdown files (not supported by myst_parser)
def reformat_md_section_links(file_path: Path):
# Read in the file
with open(file_path, "r") as file:
filedata = file.read()
# replace ".md#" by ".html#"
filedata = filedata.replace(".md#", ".html#")
filedata = re.sub(r"#(.*)\)", lambda m: m.group().lower(), filedata)
# Write the file out again
with open(file_path, "w") as file:
file.write(filedata)
for file_path in Path(".").rglob("*.md"):
reformat_md_section_links(file_path)
# -- Project information -----------------------------------------------------
project = "Substra"
copyright = f"{date.today().year}, OWKIN"
author = "Owkin"
# parse the current doc version to display it in the menu
_doc_version = re.sub("^v", "", os.popen("git describe --tags").read().strip())
# The full version, including alpha/beta/rc tags
version = _doc_version
release = _doc_version
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
"nbsphinx",
]
extensions.extend(
[
"sphinx.ext.intersphinx",
"sphinx.ext.autodoc",
"sphinx_rtd_theme",
"sphinx.ext.napoleon",
"sphinx.ext.ifconfig",
"sphinx_click",
"sphinx.ext.autosectionlabel",
"sphinx.ext.todo",
"sphinx_fontawesome",
"myst_parser", # we need it for links between md files. Recommanded by sphinx : https://www.sphinx-doc.org/en/master/usage/markdown.html
"sphinx_copybutton",
]
)
sys.path.append(os.path.abspath("./_ext"))
extensions.append("compatibilitytable")
todo_include_todos = False
intersphinx_mapping = {
"python": ("https://docs.python.org/3", None),
"numpy": ("https://numpy.org/doc/stable/", None),
"pandas": ("https://pandas.pydata.org/docs/", None),
"torch": ("https://pytorch.org/docs/stable/", None),
}
################
# Substrafl API
################
# generate autosummary even if no references
autosummary_generate = True
autosectionlabel_prefix_document = True
# autodoc settings
autodoc_default_options = {
"show-inheritance": True,
"members": True,
}
autoclass_content = "both"
autodoc_typehints = "both"
# Napoleon settings
# https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html
napoleon_google_docstring = True
napoleon_numpy_docstring = False
# Remove the prompt when copying examples
copybutton_prompt_text = ">>> "
# As we defined the type of our args, auto doc is trying to find a link to a
# documentation for each type specified
# The following elements are the link that auto doc were not able to do
nitpick_ignore = [
("py:class", "pydantic.main.BaseModel"),
("py:class", "BaseModel"),
("py:class", "FieldInfo"),
("py:class", "ConfigDict"),
("py:class", "ComputedFieldInfo"),
("py:class", "torch.nn.modules.module.Module"),
("py:class", "torch.nn.modules.Module"),
("py:class", "torch.nn.modules.loss._Loss"),
("py:class", "torch.optim.optimizer.Optimizer"),
("py:class", "torch.optim.lr_scheduler._LRScheduler"),
("py:class", "torch.utils.data.dataset.Dataset"),
("py:class", "torch.nn.modules.module.T"),
("py:class", "string"),
("py:class", "Module"),
("py:class", "optional"),
("py:class", "Dropout"),
("py:class", "BatchNorm"),
("py:class", "torch.utils.hooks.RemovableHandle"),
("py:class", "torch.nn.Parameter"),
("py:class", "Parameter"),
("py:class", "Tensor"),
("py:class", "Path"),
("py:class", "module"),
("py:attr", "persistent"),
("py:attr", "grad_input"),
("py:attr", "strict"),
("py:attr", "grad_output"),
("py:attr", "requires_grad"),
("py:attr", "device"),
("py:attr", "non_blocking"),
("py:attr", "dst_type"),
("py:attr", "dtype"),
("py:attr", "device"),
("py:attr", "assign"),
("py:func", "register_module_forward_hook"),
("py:func", "register_module_forward_pre_hook"),
("py:func", "register_module_full_backward_hook"),
("py:func", "register_module_full_backward_pre_hook"),
("py:class", "substra.sdk.schemas.Permissions"),
("py:class", "substra.Client"),
("py:class", "substra.sdk.client.Client"),
("py:class", "substra.sdk.models.ComputePlan"),
("py:class", "substra.sdk.schemas.FunctionOutputSpec"),
("py:class", "substra.sdk.schemas.FunctionInputSpec"),
("py:class", "ComputePlan"),
]
# This must be the name of an image file (path relative to the configuration
# directory) that is the favicon of the docs. Modern browsers use this as
# the icon for tabs, windows and bookmarks. It should be a Windows-style
# icon file (.ico).
html_favicon = "static/favicon.png"
# Add any paths that contain templates here, relative to this directory.
templates_path = ["templates/"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**/description.md"]
rst_epilog = f"""
.. |substra_version| replace:: {importlib.import_module('substra').__version__}
.. |substrafl_version| replace:: {importlib.import_module('substrafl').__version__}
"""
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"
html_theme = "sphinx_rtd_theme"
# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ["./static"]
html_extra_path = [str(JSON_COMPATIBILITY_TABLE_FILE)]
html_css_files = [
"owkin.css",
]
html_logo = "static/logo.svg"
html_show_sourcelink = False
html_show_sphinx = False
html_context = {
"display_github": False,
}