/
gen_gallery.py
1518 lines (1330 loc) · 54.7 KB
/
gen_gallery.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Author: Óscar Nájera
# License: 3-clause BSD
"""Sphinx-Gallery Generator.
Attaches Sphinx-Gallery to Sphinx in order to generate the galleries
when building the documentation.
"""
import codecs
import copy
from datetime import timedelta, datetime
from difflib import get_close_matches
from importlib import import_module
from pathlib import Path
import re
import os
import pathlib
from xml.sax.saxutils import quoteattr, escape
from sphinx.errors import ConfigError, ExtensionError
import sphinx.util
from sphinx.util.console import red
from . import glr_path_static, __version__ as _sg_version
from .utils import _replace_md5, _has_optipng, _has_pypandoc, _has_graphviz
from .backreferences import _finalize_backreferences
from .gen_rst import generate_dir_rst, SPHX_GLR_SIG, _get_memory_base, _get_readme
from .scrapers import _scraper_dict, _reset_dict, _import_matplotlib
from .docs_resolv import embed_code_links
from .downloads import generate_zipfiles
from .sorting import NumberOfCodeLinesSortKey
from .interactive_example import (
copy_binder_files,
check_binder_conf,
check_jupyterlite_conf,
)
from .interactive_example import pre_configure_jupyterlite_sphinx
from .interactive_example import post_configure_jupyterlite_sphinx
from .interactive_example import create_jupyterlite_contents
from .directives import MiniGallery, ImageSg, imagesg_addnode
_KNOWN_CSS = (
"sg_gallery",
"sg_gallery-binder",
"sg_gallery-dataframe",
"sg_gallery-rendered-html",
)
class DefaultResetArgv:
"""Provides default 'reset_argv' callable that returns empty list."""
def __repr__(self):
return "DefaultResetArgv"
def __call__(self, gallery_conf, script_vars):
"""Return empty list."""
return []
DEFAULT_GALLERY_CONF = {
"filename_pattern": re.escape(os.sep) + "plot",
"ignore_pattern": r"__init__\.py",
"examples_dirs": os.path.join("..", "examples"),
"example_extensions": {".py"},
"filetype_parsers": {},
"notebook_extensions": {".py"},
"reset_argv": DefaultResetArgv(),
"subsection_order": None,
"within_subsection_order": NumberOfCodeLinesSortKey,
"gallery_dirs": "auto_examples",
"backreferences_dir": None,
"doc_module": (),
"exclude_implicit_doc": set(),
"reference_url": {},
"capture_repr": ("_repr_html_", "__repr__"),
"ignore_repr_types": r"",
# Build options
# -------------
# 'plot_gallery' also accepts strings that evaluate to a bool, e.g. "True",
# "False", "1", "0" so that they can be easily set via command line
# switches of sphinx-build
"plot_gallery": "True",
"download_all_examples": True,
"abort_on_example_error": False,
"only_warn_on_example_error": False,
"failing_examples": {},
"passing_examples": [],
"stale_examples": [], # ones that did not need to be run due to md5sum
"run_stale_examples": False,
"expected_failing_examples": set(),
"thumbnail_size": (400, 280), # Default CSS does 0.4 scaling (160, 112)
"min_reported_time": 0,
"binder": {},
"jupyterlite": {},
"promote_jupyter_magic": False,
"image_scrapers": ("matplotlib",),
"compress_images": (),
"reset_modules": ("matplotlib", "seaborn"),
"reset_modules_order": "before",
"first_notebook_cell": None,
"last_notebook_cell": None,
"notebook_images": False,
"pypandoc": False,
"remove_config_comments": False,
"show_memory": False,
"show_signature": True,
"junit": "",
"log_level": {"backreference_missing": "warning"},
"inspect_global_variables": True,
"css": _KNOWN_CSS,
"matplotlib_animations": False,
"image_srcset": [],
"default_thumb_file": None,
"line_numbers": False,
"nested_sections": True,
"prefer_full_module": set(),
"api_usage_ignore": ".*__.*__",
"show_api_usage": False, # if this changes, change write_api_entries, too
"copyfile_regex": "",
}
logger = sphinx.util.logging.getLogger("sphinx-gallery")
def _bool_eval(x):
"""Evaluate bool only configs, to allow setting via -D on the command line."""
if isinstance(x, str):
try:
x = eval(x)
except TypeError:
pass
return bool(x)
def _update_gallery_conf_exclude_implicit_doc(gallery_conf):
"""Update gallery config exclude_implicit_doc.
This is separate function for better testability.
"""
# prepare regex for exclusions from implicit documentation
exclude_regex = (
re.compile("|".join(gallery_conf["exclude_implicit_doc"]))
if gallery_conf["exclude_implicit_doc"]
else False
)
gallery_conf["exclude_implicit_doc_regex"] = exclude_regex
def _update_gallery_conf_builder_inited(
sphinx_gallery_conf,
src_dir,
plot_gallery=True,
abort_on_example_error=False,
builder_name="html",
):
sphinx_gallery_conf.update(plot_gallery=plot_gallery)
sphinx_gallery_conf.update(abort_on_example_error=abort_on_example_error)
sphinx_gallery_conf["src_dir"] = src_dir
# Make it easy to know which builder we're in
sphinx_gallery_conf["builder_name"] = builder_name
def _fill_gallery_conf_defaults(sphinx_gallery_conf, app=None, check_keys=True):
"""Handle user configs and update default gallery configs."""
gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF)
options = sorted(gallery_conf)
extra_keys = sorted(set(sphinx_gallery_conf) - set(options))
if extra_keys and check_keys:
msg = "Unknown key(s) in sphinx_gallery_conf:\n"
for key in extra_keys:
options = get_close_matches(key, options, cutoff=0.66)
msg += repr(key)
if len(options) == 1:
msg += f", did you mean {options[0]!r}?"
elif len(options) > 1:
msg += f", did you mean one of {options!r}?"
msg += "\n"
raise ConfigError(msg.strip())
gallery_conf.update(sphinx_gallery_conf)
# XXX anything that can only be a bool (rather than str) should probably be
# evaluated this way as it allows setting via -D on the command line
for key in (
"promote_jupyter_magic",
"run_stale_examples",
):
gallery_conf[key] = _bool_eval(gallery_conf[key])
gallery_conf["app"] = app
# Check capture_repr
capture_repr = gallery_conf["capture_repr"]
supported_reprs = ["__repr__", "__str__", "_repr_html_"]
if isinstance(capture_repr, tuple):
for rep in capture_repr:
if rep not in supported_reprs:
raise ConfigError(
"All entries in 'capture_repr' must be one "
f"of {supported_reprs}, got: {rep}"
)
else:
raise ConfigError(f"'capture_repr' must be a tuple, got: {type(capture_repr)}")
# Check ignore_repr_types
if not isinstance(gallery_conf["ignore_repr_types"], str):
raise ConfigError(
"'ignore_repr_types' must be a string, got: "
+ type(gallery_conf["ignore_repr_types"])
)
# deal with show_memory
gallery_conf["memory_base"] = 0.0
if gallery_conf["show_memory"]:
if not callable(gallery_conf["show_memory"]): # True-like
try:
from memory_profiler import memory_usage # noqa
except ImportError:
logger.warning(
"Please install 'memory_profiler' to enable "
"peak memory measurements."
)
gallery_conf["show_memory"] = False
else:
def call_memory(func):
mem, out = memory_usage(
func, max_usage=True, retval=True, multiprocess=True
)
try:
mem = mem[0] # old MP always returned a list
except TypeError: # 'float' object is not subscriptable
pass
return mem, out
gallery_conf["call_memory"] = call_memory
gallery_conf["memory_base"] = _get_memory_base(gallery_conf)
else:
gallery_conf["call_memory"] = gallery_conf["show_memory"]
if not gallery_conf["show_memory"]: # can be set to False above
def call_memory(func):
return 0.0, func()
gallery_conf["call_memory"] = call_memory
assert callable(gallery_conf["call_memory"])
# deal with scrapers
scrapers = gallery_conf["image_scrapers"]
if not isinstance(scrapers, (tuple, list)):
scrapers = [scrapers]
scrapers = list(scrapers)
for si, scraper in enumerate(scrapers):
if isinstance(scraper, str):
if scraper in _scraper_dict:
scraper = _scraper_dict[scraper]
else:
orig_scraper = scraper
try:
scraper = import_module(scraper)
scraper = getattr(scraper, "_get_sg_image_scraper")
scraper = scraper()
except Exception as exp:
raise ConfigError(
f"Unknown image scraper {orig_scraper!r}, got:\n{exp}"
)
scrapers[si] = scraper
if not callable(scraper):
raise ConfigError(f"Scraper {scraper!r} was not callable")
gallery_conf["image_scrapers"] = tuple(scrapers)
del scrapers
# Here we try to set up matplotlib but don't raise an error,
# we will raise an error later when we actually try to use it
# (if we do so) in scrapers.py.
# In principle we could look to see if there is a matplotlib scraper
# in our scrapers list, but this would be backward incompatible with
# anyone using or relying on our Agg-setting behavior (e.g., for some
# custom matplotlib SVG scraper as in our docs).
# Eventually we can make this a config var like matplotlib_agg or something
# if people need us not to set it to Agg.
try:
_import_matplotlib()
except (ImportError, ValueError):
pass
# compress_images
compress_images = gallery_conf["compress_images"]
if isinstance(compress_images, str):
compress_images = [compress_images]
elif not isinstance(compress_images, (tuple, list)):
raise ConfigError(
"compress_images must be a tuple, list, or str, "
f"got {type(compress_images)}"
)
compress_images = list(compress_images)
allowed_values = ("images", "thumbnails")
pops = list()
for ki, kind in enumerate(compress_images):
if kind not in allowed_values:
if kind.startswith("-"):
pops.append(ki)
continue
raise ConfigError(
"All entries in compress_images must be one of "
f"{allowed_values} or a command-line switch "
f'starting with "-", got {kind!r}'
)
compress_images_args = [compress_images.pop(p) for p in pops[::-1]]
if len(compress_images) and not _has_optipng():
logger.warning(
"optipng binaries not found, PNG %s will not be optimized",
" and ".join(compress_images),
)
compress_images = ()
gallery_conf["compress_images"] = compress_images
gallery_conf["compress_images_args"] = compress_images_args
# deal with resetters
resetters = gallery_conf["reset_modules"]
if not isinstance(resetters, (tuple, list)):
resetters = [resetters]
resetters = list(resetters)
for ri, resetter in enumerate(resetters):
if isinstance(resetter, str):
if resetter not in _reset_dict:
raise ConfigError(f"Unknown module resetter named {resetter!r}")
resetters[ri] = _reset_dict[resetter]
elif not callable(resetter):
raise ConfigError(f"Module resetter {resetter!r} was not callable")
gallery_conf["reset_modules"] = tuple(resetters)
if not isinstance(gallery_conf["reset_modules_order"], str):
raise ConfigError(
"reset_modules_order must be a str, "
f'got {gallery_conf["reset_modules_order"]!r}'
)
if gallery_conf["reset_modules_order"] not in ["before", "after", "both"]:
raise ConfigError(
"reset_modules_order must be in"
"['before', 'after', 'both'], "
f"got {gallery_conf['reset_modules_order']!r}"
)
del resetters
# Ensure the first cell text is a string if we have it
first_cell = gallery_conf.get("first_notebook_cell")
if (not isinstance(first_cell, str)) and (first_cell is not None):
raise ConfigError(
"The 'first_notebook_cell' parameter must be type "
f"str or None, found type {type(first_cell)}"
)
# Ensure the last cell text is a string if we have it
last_cell = gallery_conf.get("last_notebook_cell")
if (not isinstance(last_cell, str)) and (last_cell is not None):
raise ConfigError(
"The 'last_notebook_cell' parameter must be type str"
f" or None, found type {type(last_cell)}"
)
# Check pypandoc
pypandoc = gallery_conf["pypandoc"]
if not isinstance(pypandoc, (dict, bool)):
raise ConfigError(
"'pypandoc' parameter must be of type bool or dict,"
f"got: {type(pypandoc)}."
)
gallery_conf["pypandoc"] = dict() if pypandoc is True else pypandoc
has_pypandoc, version = _has_pypandoc()
if isinstance(gallery_conf["pypandoc"], dict) and has_pypandoc is None:
logger.warning(
"'pypandoc' not available. Using Sphinx-Gallery to "
"convert rst text blocks to markdown for .ipynb files."
)
gallery_conf["pypandoc"] = False
elif isinstance(gallery_conf["pypandoc"], dict):
logger.info(
"Using pandoc version: %s to convert rst text blocks to "
"markdown for .ipynb files",
version,
)
else:
logger.info(
"Using Sphinx-Gallery to convert rst text blocks to "
"markdown for .ipynb files."
)
if isinstance(pypandoc, dict):
accepted_keys = ("extra_args", "filters")
for key in pypandoc:
if key not in accepted_keys:
raise ConfigError(
"'pypandoc' only accepts the following key "
f"values: {accepted_keys}, got: {key}."
)
gallery_conf["titles"] = {}
# Ensure 'backreferences_dir' is str, pathlib.Path or None
backref = gallery_conf["backreferences_dir"]
if (not isinstance(backref, (str, pathlib.Path))) and (backref is not None):
raise ConfigError(
"The 'backreferences_dir' parameter must be of type "
"str, pathlib.Path or None, "
f"found type {type(backref)}"
)
# if 'backreferences_dir' is pathlib.Path, make str for Python <=3.5
# compatibility
if isinstance(backref, pathlib.Path):
gallery_conf["backreferences_dir"] = str(backref)
# binder
gallery_conf["binder"] = check_binder_conf(gallery_conf["binder"])
# jupyterlite
gallery_conf["jupyterlite"] = check_jupyterlite_conf(
gallery_conf.get("jupyterlite", {}), app
)
if not isinstance(gallery_conf["css"], (list, tuple)):
raise ConfigError(
'gallery_conf["css"] must be list or tuple, got ' f'{gallery_conf["css"]!r}'
)
for css in gallery_conf["css"]:
if css not in _KNOWN_CSS:
raise ConfigError(f"Unknown css {css!r}, must be one of {_KNOWN_CSS!r}")
if gallery_conf["app"] is not None: # can be None in testing
gallery_conf["app"].add_css_file(css + ".css")
# check API usage
if not isinstance(gallery_conf["api_usage_ignore"], str):
raise ConfigError(
'gallery_conf["api_usage_ignore"] must be str, '
f'got {type(gallery_conf["api_usage_ignore"])}'
)
if (
not isinstance(gallery_conf["show_api_usage"], bool)
and gallery_conf["show_api_usage"] != "unused"
):
raise ConfigError(
'gallery_conf["show_api_usage"] must be True, False or "unused", '
f'got {gallery_conf["show_api_usage"]}'
)
_update_gallery_conf_exclude_implicit_doc(gallery_conf)
return gallery_conf
def get_subsections(srcdir, examples_dir, gallery_conf, check_for_index=True):
"""Return the list of subsections of a gallery.
Parameters
----------
srcdir : str
absolute path to directory containing conf.py
examples_dir : str
path to the examples directory relative to conf.py
gallery_conf : Dict[str, Any]
Sphinx-Gallery configuration dictionary.
check_for_index : bool
only return subfolders with a ReadMe, default True
Returns
-------
out : list
sorted list of gallery subsection folder names
"""
sortkey = gallery_conf["subsection_order"]
subfolders = [subfolder for subfolder in os.listdir(examples_dir)]
if check_for_index:
subfolders = [
subfolder
for subfolder in subfolders
if _get_readme(
os.path.join(examples_dir, subfolder), gallery_conf, raise_error=False
)
is not None
]
else:
# just make sure its a directory
subfolders = [
subfolder
for subfolder in subfolders
if os.path.isdir(os.path.join(examples_dir, subfolder))
]
base_examples_dir_path = os.path.relpath(examples_dir, srcdir)
subfolders_with_path = [
os.path.join(base_examples_dir_path, item) for item in subfolders
]
sorted_subfolders = sorted(subfolders_with_path, key=sortkey)
return [
subfolders[i]
for i in [subfolders_with_path.index(item) for item in sorted_subfolders]
]
def _prepare_sphx_glr_dirs(gallery_conf, srcdir):
"""Creates necessary folders for sphinx_gallery files."""
examples_dirs = gallery_conf["examples_dirs"]
gallery_dirs = gallery_conf["gallery_dirs"]
if not isinstance(examples_dirs, list):
examples_dirs = [examples_dirs]
if not isinstance(gallery_dirs, list):
gallery_dirs = [gallery_dirs]
if bool(gallery_conf["backreferences_dir"]):
backreferences_dir = os.path.join(srcdir, gallery_conf["backreferences_dir"])
if not os.path.exists(backreferences_dir):
os.makedirs(backreferences_dir)
return list(zip(examples_dirs, gallery_dirs))
def _format_toctree(items, includehidden=False):
"""Format a toc tree."""
st = """
.. toctree::
:hidden:"""
if includehidden:
st += """
:includehidden:
"""
st += """
{}\n""".format(
"\n ".join(items)
)
st += "\n"
return st
def generate_gallery_rst(app):
"""Generate the Main examples gallery reStructuredText.
Start the Sphinx-Gallery configuration and recursively scan the examples
directories in order to populate the examples gallery.
We create a 2-level nested structure by iterating through every
sibling folder of the current index file.
In each of these folders, we look for a section index file,
for which we generate a toctree pointing to sibling scripts.
Then, we append the content of this section index file
to the current index file,
after we remove toctree (to keep a clean nested structure)
and sphinx tags (to prevent tag duplication)
Eventually, we create a toctree in the current index file
which points to section index files.
"""
logger.info("generating gallery...", color="white")
gallery_conf = app.config.sphinx_gallery_conf
seen_backrefs = set()
costs = []
workdirs = _prepare_sphx_glr_dirs(gallery_conf, app.builder.srcdir)
# Check for duplicate filenames to make sure linking works as expected
examples_dirs = [ex_dir for ex_dir, _ in workdirs]
files = collect_gallery_files(examples_dirs, gallery_conf)
check_duplicate_filenames(files)
check_spaces_in_filenames(files)
for examples_dir, gallery_dir in workdirs:
examples_dir_abs_path = os.path.join(app.builder.srcdir, examples_dir)
gallery_dir_abs_path = os.path.join(app.builder.srcdir, gallery_dir)
# Create section rst files and fetch content which will
# be added to current index file. This only includes content
# from files located in the root folder of the current gallery
# (ie not in subfolders)
(
_,
this_content,
this_costs,
this_toctree_items,
) = generate_dir_rst(
examples_dir_abs_path,
gallery_dir_abs_path,
gallery_conf,
seen_backrefs,
include_toctree=False,
)
has_readme = this_content is not None
costs += this_costs
write_computation_times(gallery_conf, gallery_dir_abs_path, this_costs)
# We create an index.rst with all examples
# (this will overwrite the rst file generated by the previous call
# to generate_dir_rst)
if this_content:
# :orphan: to suppress "not included in TOCTREE" sphinx warnings
indexst = ":orphan:\n\n" + this_content
else:
# we are not going to use the index.rst.new that gets made here,
# but go through the motions to run through all the subsections...
indexst = "Never used!"
# Write toctree with gallery items from gallery root folder
if len(this_toctree_items) > 0:
this_toctree = _format_toctree(this_toctree_items)
indexst += this_toctree
# list all paths to subsection index files in this array
subsection_index_files = []
subsecs = get_subsections(
app.builder.srcdir,
examples_dir_abs_path,
gallery_conf,
check_for_index=has_readme,
)
for subsection in subsecs:
src_dir = os.path.join(examples_dir_abs_path, subsection)
target_dir = os.path.join(gallery_dir_abs_path, subsection)
subsection_index_files.append(
"/".join(["", gallery_dir, subsection, "index.rst"]).replace(
os.sep, "/"
) # fwd slashes needed in rst
)
(
subsection_index_path,
subsection_index_content,
subsection_costs,
subsection_toctree_filenames,
) = generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs)
if subsection_index_content:
# Filter out tags from subsection content
# to prevent tag duplication across the documentation
tag_regex = r"^\.\.(\s+)\_(.+)\:(\s*)$"
subsection_index_content = "\n".join(
[
line
for line in subsection_index_content.splitlines()
if re.match(tag_regex, line) is None
]
+ [""]
)
indexst += subsection_index_content
has_readme_subsection = True
else:
has_readme_subsection = False
# Write subsection toctree in main file only if
# nested_sections is False or None, and
# toctree filenames were generated for the subsection.
if not gallery_conf["nested_sections"]:
if len(subsection_toctree_filenames) > 0:
subsection_index_toctree = _format_toctree(
subsection_toctree_filenames
)
indexst += subsection_index_toctree
# Otherwise, a new index.rst.new file should
# have been created and it needs to be parsed
elif has_readme_subsection:
_replace_md5(subsection_index_path, mode="t")
costs += subsection_costs
write_computation_times(gallery_conf, target_dir, subsection_costs)
# generate toctree with subsections
if gallery_conf["nested_sections"] is True:
subsections_toctree = _format_toctree(
subsection_index_files, includehidden=True
)
# add toctree to file only if there are subsections
if len(subsection_index_files) > 0:
indexst += subsections_toctree
if gallery_conf["download_all_examples"]:
download_fhindex = generate_zipfiles(
gallery_dir_abs_path, app.builder.srcdir, gallery_conf
)
indexst += download_fhindex
if app.config.sphinx_gallery_conf["show_signature"]:
indexst += SPHX_GLR_SIG
if has_readme:
index_rst_new = os.path.join(gallery_dir_abs_path, "index.rst.new")
with codecs.open(index_rst_new, "w", encoding="utf-8") as fhindex:
fhindex.write(indexst)
_replace_md5(index_rst_new, mode="t")
# Write a single global sg_execution_times
write_computation_times(gallery_conf, None, costs)
if gallery_conf["show_api_usage"] is not False:
_init_api_usage(app.builder.srcdir)
_finalize_backreferences(seen_backrefs, gallery_conf)
if gallery_conf["plot_gallery"]:
logger.info("computation time summary:", color="white")
lines, lens = _format_for_writing(
costs, src_dir=gallery_conf["src_dir"], kind="console"
)
for name, t, m in lines:
text = (f" - {name}: ").ljust(lens[0] + 10)
if t is None:
text += "(not run)"
logger.info(text)
else:
t_float = float(t.split()[0])
if t_float >= gallery_conf["min_reported_time"]:
text += t.rjust(lens[1]) + " " + m.rjust(lens[2])
logger.info(text)
# Also create a junit.xml file, useful e.g. on CircleCI
write_junit_xml(gallery_conf, app.builder.outdir, costs)
SPHX_GLR_ORPHAN = """
:orphan:
.. _{0}:
"""
SPHX_GLR_COMP_TIMES = (
SPHX_GLR_ORPHAN
+ """
Computation times
=================
"""
)
def _sec_to_readable(t):
"""Convert a number of seconds to a more readable representation."""
# This will only work for < 1 day execution time
# And we reserve 2 digits for minutes because presumably
# there aren't many > 99 minute scripts, but occasionally some
# > 9 minute ones
t = datetime(1, 1, 1) + timedelta(seconds=t)
t = "{:02d}:{:02d}.{:03d}".format(
t.hour * 60 + t.minute, t.second, int(round(t.microsecond / 1000.0))
)
return t
def _cost_key(cost):
"""Cost sorting function."""
# sort by descending computation time, descending memory, alphabetical name
return (-cost["t"], -cost["mem"], cost["src_file"])
def _format_for_writing(costs, *, src_dir, kind="rst"):
"""Provide formatted computation summary text.
Parameters
----------
costs: List[Dict]
List of dicts of computation costs and paths, see gen_rst.py for details.
src_dir : pathlib.Path
The Sphinx source directory.
kind: 'rst', 'rst-full' or 'console', default='rst'
Format for printing to 'console' or for writing `sg_execution_times.rst' ('rst'
for single galleries and 'rst-full' for all galleries).
Returns
-------
lines: List[List[str]]
Formatted computation text for each example, of format:
[example_file, time_elapsed, memory_used]
lens: List[int]
Character length of each string in `lines`.
"""
lines = list()
for cost in sorted(costs, key=_cost_key):
src_file = cost["src_file"]
rel_path = os.path.relpath(src_file, src_dir)
if kind in ("rst", "rst-full"): # like in sg_execution_times
target_dir_clean = os.path.relpath(cost["target_dir"], src_dir).replace(
os.path.sep, "_"
)
paren = rel_path if kind == "rst-full" else os.path.basename(src_file)
name = ":ref:`sphx_glr_{0}_{1}` (``{2}``)".format(
target_dir_clean, os.path.basename(src_file), paren
)
t = _sec_to_readable(cost["t"])
else: # like in generate_gallery
assert kind == "console"
name = rel_path
t = f'{cost["t"]:0.2f} sec'
m = f'{cost["mem"]:.1f} MB'
lines.append([name, t, m])
lens = [max(x) for x in zip(*[[len(item) for item in cost] for cost in lines])]
return lines, lens
def write_computation_times(gallery_conf, target_dir, costs):
"""Write computation times to `sg_execution_times.rst`.
Parameters
----------
gallery_conf : Dict[str, Any]
Sphinx-Gallery configuration dictionary.
target_dir : str | None
Path to directory where example python source file are.
costs: List[Dict]
List of dicts of computation costs and paths, see gen_rst.py for details.
"""
total_time = sum(cost["t"] for cost in costs)
if target_dir is None: # all galleries together
out_dir = gallery_conf["src_dir"]
where = "all galleries"
kind = "rst-full"
ref_extra = ""
else: # a single gallery
out_dir = target_dir
where = os.path.relpath(target_dir, gallery_conf["src_dir"])
kind = "rst"
ref_extra = f'{where.replace(os.path.sep, "_")}_'
new_ref = f"sphx_glr_{ref_extra}sg_execution_times"
out_file = Path(out_dir) / "sg_execution_times.rst"
if out_file.is_file() and total_time == 0: # a re-run
return
with out_file.open("w", encoding="utf-8") as fid:
fid.write(SPHX_GLR_COMP_TIMES.format(new_ref))
fid.write(
f"**{_sec_to_readable(total_time)}** total execution time for "
f"{len(costs)} file{'s' if len(costs) != 1 else ''} **from {where}**:\n\n"
)
lines, lens = _format_for_writing(
costs,
src_dir=gallery_conf["src_dir"],
kind=kind,
)
del costs
# https://datatables.net/examples/styling/bootstrap5.html
fid.write( # put it in a container to make the scoped style work
"""\
.. container::
.. raw:: html
<style scoped>
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/css/bootstrap.min.css" rel="stylesheet" />
<link href="https://cdn.datatables.net/1.13.6/css/dataTables.bootstrap5.min.css" rel="stylesheet" />
</style>
<script src="https://code.jquery.com/jquery-3.7.0.js"></script>
<script src="https://cdn.datatables.net/1.13.6/js/jquery.dataTables.min.js"></script>
<script src="https://cdn.datatables.net/1.13.6/js/dataTables.bootstrap5.min.js"></script>
<script type="text/javascript" class="init">
$(document).ready( function () {
$('table.sg-datatable').DataTable({order: [[1, 'desc']]});
} );
</script>
.. list-table::
:header-rows: 1
:class: table table-striped sg-datatable
* - Example
- Time
- Mem (MB)
""" # noqa: E501
)
# Need at least one entry or Sphinx complains
for ex, t, mb in lines or [["N/A", "N/A", "N/A"]]:
fid.write(
f"""\
* - {ex}
- {t}
- {mb.rsplit(maxsplit=1)[0]}
"""
) # remove the "MB" from the right
def write_api_entries(app, what, name, obj, options, lines):
"""Write api entries to `_sg_api_entries` configuration.
To connect to `autodoc-process-docstring` event.
Parameters
----------
app :
The Sphinx application object.
what: str
The type of the object which the docstring belongs to. One of
"module", "class", "exception", "function", "method", "attribute".
name :
The fully qualified name of the object.
obj :
The object itself.
options :
The options given to the directive: an object with attributes inherited_members,
undoc_members, show_inheritance and no-index that are true if the flag option of
same name was given to the auto directive.
lines :
The lines of the docstring, see above.
"""
if app.config.sphinx_gallery_conf["show_api_usage"] is False:
return
if "_sg_api_entries" not in app.config.sphinx_gallery_conf:
app.config.sphinx_gallery_conf["_sg_api_entries"] = dict()
if what not in app.config.sphinx_gallery_conf["_sg_api_entries"]:
app.config.sphinx_gallery_conf["_sg_api_entries"][what] = set()
app.config.sphinx_gallery_conf["_sg_api_entries"][what].add(name)
def _init_api_usage(gallery_dir):
with codecs.open(
os.path.join(gallery_dir, "sg_api_usage.rst"), "w", encoding="utf-8"
):
pass
# Colors from https://personal.sron.nl/~pault/data/colourschemes.pdf
# 3 Diverging Colour Schemes, Figure 12, plus alpha=AA
API_COLORS = dict(
edge="#00000080", # gray (by alpha)
okay="#98CAE180", # blue
bad_1="#FEDA8B80", # yellow
bad_2="#F67E4B80", # orange
bad_3="#A5002680", # red
)
def _make_graph(fname, entries, gallery_conf):
"""Make a graph of unused and used API entries.
The used API entries themselves are documented in the list, so
for the graph, we'll focus on the number of unused API entries
per modules. Modules with lots of unused entries (11+) will be colored
red, those with less (6+) will be colored orange, those with only a few
(1-5) will be colored yellow and those with no unused entries will be
colored blue.
The API entries that are used are shown with one graph per module.
That way you can see the examples that each API entry is used in
for that module (if this was done for the whole project at once,
the graph would get too large very large quickly).
Parameters
----------
fname: str
Path to '*sg_api_unused.dot' file.
entries: Dict[str, List] or List[str]
Used (List) or unused (Dict) API entries.
gallery_conf : Dict[str, Any]
Sphinx-Gallery configuration dictionary.
"""
import graphviz
dg = graphviz.Digraph(
filename=fname,
graph_attr={
"overlap": "scale",
"pad": "0.5",
},
node_attr={
"color": API_COLORS["okay"],
"style": "filled",
"fontsize": "20",
"shape": "box",
"fontname": "Open Sans,Arial",
},
)
if isinstance(entries, list):
connections = set()
lut = dict() # look up table for connections so they don't repeat
structs = [entry.split(".") for entry in entries]
for struct in sorted(structs, key=len):
for level in range(len(struct) - 2):
if (struct[level], struct[level + 1]) in connections:
continue
connections.add((struct[level], struct[level + 1]))
node_from = (
lut[struct[level]] if struct[level] in lut else struct[level]
)
dg.node(node_from)
node_to = struct[level + 1]
node_kwargs = dict()
# count, don't show leaves
if len(struct) - 3 == level:
leaf_count = 0
for struct2 in structs:
# find structures of the same length as struct
if len(struct2) != level + 3:
continue
# find structures with two entries before
# the leaf that are the same as struct
if all(
[
struct2[level2] == struct[level2]
for level2 in range(level + 2)
]
):
leaf_count += 1
node_to += f" ({leaf_count})"
lut[struct[level + 1]] = node_to
if leaf_count > 10:
color_key = "bad_3"