diff --git a/MANIFEST.in b/MANIFEST.in index cb31a0e4..69533d73 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ # data include templateflow/conf/config.json include templateflow/conf/templateflow-skel.zip +include templateflow/conf/templateflow-skel.md5 # misc include requirements.txt diff --git a/docs/conf.py b/docs/conf.py index 4d66209a..f2c0232a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,12 +16,12 @@ from templateflow import __version__, __copyright__, __packagename__ -sys.path.append(os.path.abspath('sphinxext')) +sys.path.append(os.path.abspath("sphinxext")) # -- Project information ----------------------------------------------------- project = __packagename__ copyright = __copyright__ -author = 'The TemplateFlow Developers' +author = "The TemplateFlow Developers" # The short X.Y version version = Version(__version__).public @@ -31,29 +31,29 @@ # -- General configuration --------------------------------------------------- extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinxcontrib.apidoc', - 'sphinxcontrib.napoleon' + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinxcontrib.apidoc", + "sphinxcontrib.napoleon", ] autodoc_mock_imports = [ - 'matplotlib', - 'nilearn', - 'nipy', - 'nitime', - 'numpy', - 'pandas', - 'seaborn', - 'skimage', - 'svgutils', - 'transforms3d', + "matplotlib", + "nilearn", + "nipy", + "nitime", + "numpy", + "pandas", + "seaborn", + "skimage", + "svgutils", + "transforms3d", ] # Accept custom section names to be parsed for numpy-style docstrings @@ -62,21 +62,21 @@ # https://github.com/sphinx-contrib/napoleon/pull/10 is merged. napoleon_use_param = False napoleon_custom_sections = [ - ('Inputs', 'Parameters'), - ('Outputs', 'Parameters'), + ("Inputs", "Parameters"), + ("Outputs", "Parameters"), ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -89,8 +89,12 @@ # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [ - '_build', 'Thumbs.db', '.DS_Store', - 'api/modules.rst', 'api/templateflow.rst'] + "_build", + "Thumbs.db", + ".DS_Store", + "api/modules.rst", + "api/templateflow.rst", +] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -101,7 +105,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -112,7 +116,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -128,7 +132,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'templateflowdoc' +htmlhelp_basename = "templateflowdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -137,15 +141,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -155,8 +156,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'templateflow.tex', 'TemplateFlow Documentation', - 'The TemplateFlow Developers', 'manual'), + ( + master_doc, + "templateflow.tex", + "TemplateFlow Documentation", + "The TemplateFlow Developers", + "manual", + ), ] @@ -164,10 +170,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'templateflow', 'TemplateFlow Documentation', - [author], 1) -] +man_pages = [(master_doc, "templateflow", "TemplateFlow Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -176,9 +179,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'templateflow', 'TemplateFlow Documentation', - author, 'TemplateFlow', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "templateflow", + "TemplateFlow Documentation", + author, + "TemplateFlow", + "One line description of project.", + "Miscellaneous", + ), ] @@ -197,21 +206,21 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- -apidoc_module_dir = '../templateflow' -apidoc_output_dir = 'api' -apidoc_excluded_paths = ['conftest.py', '*/tests/*', 'tests/*', 'data/*'] +apidoc_module_dir = "../templateflow" +apidoc_output_dir = "api" +apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "data/*"] apidoc_separate_modules = True -apidoc_extra_args = ['--module-first', '-d 1', '-T'] +apidoc_extra_args = ["--module-first", "-d 1", "-T"] # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {"https://docs.python.org/": None} # -- Options for versioning extension ---------------------------------------- scv_show_banner = True diff --git a/docs/sphinxext/docscrape.py b/docs/sphinxext/docscrape.py index 470badd3..b4e0a555 100644 --- a/docs/sphinxext/docscrape.py +++ b/docs/sphinxext/docscrape.py @@ -16,6 +16,7 @@ class Reader(object): """A line-based string reader. """ + def __init__(self, data): """ Parameters @@ -27,7 +28,7 @@ def __init__(self, data): if isinstance(data, list): self._str = data else: - self._str = data.split('\n') # store string as list of lines + self._str = data.split("\n") # store string as list of lines self.reset() @@ -43,10 +44,10 @@ def read(self): self._l += 1 return out else: - return '' + return "" def seek_next_non_empty_line(self): - for l in self[self._l:]: + for l in self[self._l :]: if l.strip(): break else: @@ -59,10 +60,10 @@ def read_to_condition(self, condition_func): start = self._l for line in self[start:]: if condition_func(line): - return self[start:self._l] + return self[start : self._l] self._l += 1 if self.eof(): - return self[start:self._l+1] + return self[start : self._l + 1] return [] def read_to_next_empty_line(self): @@ -75,43 +76,44 @@ def is_empty(line): def read_to_next_unindented_line(self): def is_unindented(line): - return (line.strip() and (len(line.lstrip()) == len(line))) + return line.strip() and (len(line.lstrip()) == len(line)) + return self.read_to_condition(is_unindented) def peek(self, n=0): if self._l + n < len(self._str): return self[self._l + n] else: - return '' + return "" def is_empty(self): - return not ''.join(self._str).strip() + return not "".join(self._str).strip() class NumpyDocString(collections.Mapping): def __init__(self, docstring, config={}): - docstring = textwrap.dedent(docstring).split('\n') + docstring = textwrap.dedent(docstring).split("\n") self._doc = Reader(docstring) self._parsed_data = { - 'Signature': '', - 'Summary': [''], - 'Extended Summary': [], - 'Parameters': [], - 'Returns': [], - 'Yields': [], - 'Raises': [], - 'Warns': [], - 'Other Parameters': [], - 'Attributes': [], - 'Methods': [], - 'See Also': [], - 'Notes': [], - 'Warnings': [], - 'References': '', - 'Examples': '', - 'index': {} - } + "Signature": "", + "Summary": [""], + "Extended Summary": [], + "Parameters": [], + "Returns": [], + "Yields": [], + "Raises": [], + "Warns": [], + "Other Parameters": [], + "Attributes": [], + "Methods": [], + "See Also": [], + "Notes": [], + "Warnings": [], + "References": "", + "Examples": "", + "index": {}, + } self._parse() @@ -138,11 +140,11 @@ def _is_at_section(self): l1 = self._doc.peek().strip() # e.g. Parameters - if l1.startswith('.. index::'): + if l1.startswith(".. index::"): return True l2 = self._doc.peek(1).strip() # ---------- or ========== - return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1)) + return l2.startswith("-" * len(l1)) or l2.startswith("=" * len(l1)) def _strip(self, doc): i = 0 @@ -155,14 +157,14 @@ def _strip(self, doc): if line.strip(): break - return doc[i:len(doc)-j] + return doc[i : len(doc) - j] def _read_to_next_section(self): section = self._doc.read_to_next_empty_line() while not self._is_at_section() and not self._doc.eof(): if not self._doc.peek(-1).strip(): # previous line was empty - section += [''] + section += [""] section += self._doc.read_to_next_empty_line() @@ -173,7 +175,7 @@ def _read_sections(self): data = self._read_to_next_section() name = data[0].strip() - if name.startswith('..'): # index section + if name.startswith(".."): # index section yield name, data[1:] elif len(data) < 2: yield StopIteration @@ -185,10 +187,10 @@ def _parse_param_list(self, content): params = [] while not r.eof(): header = r.read().strip() - if ' : ' in header: - arg_name, arg_type = header.split(' : ')[:2] + if " : " in header: + arg_name, arg_type = header.split(" : ")[:2] else: - arg_name, arg_type = header, '' + arg_name, arg_type = header, "" desc = r.read_to_next_unindented_line() desc = dedent_lines(desc) @@ -197,8 +199,11 @@ def _parse_param_list(self, content): return params - _name_rgx = re.compile(r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" - r" (?P[a-zA-Z0-9_.-]+))\s*", re.X) + _name_rgx = re.compile( + r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" + r" (?P[a-zA-Z0-9_.-]+))\s*", + re.X, + ) def _parse_see_also(self, content): """ @@ -236,17 +241,17 @@ def push_item(name, rest): continue m = self._name_rgx.match(line) - if m and line[m.end():].strip().startswith(':'): + if m and line[m.end() :].strip().startswith(":"): push_item(current_func, rest) - current_func, line = line[:m.end()], line[m.end():] - rest = [line.split(':', 1)[1].strip()] + current_func, line = line[: m.end()], line[m.end() :] + rest = [line.split(":", 1)[1].strip()] if not rest[0]: rest = [] - elif not line.startswith(' '): + elif not line.startswith(" "): push_item(current_func, rest) current_func = None - if ',' in line: - for func in line.split(','): + if "," in line: + for func in line.split(","): if func.strip(): push_item(func, []) elif line.strip(): @@ -262,17 +267,18 @@ def _parse_index(self, section, content): :refguide: something, else, and more """ + def strip_each_in(lst): return [s.strip() for s in lst] out = {} - section = section.split('::') + section = section.split("::") if len(section) > 1: - out['default'] = strip_each_in(section[1].split(','))[0] + out["default"] = strip_each_in(section[1].split(","))[0] for line in content: - line = line.split(':') + line = line.split(":") if len(line) > 2: - out[line[1]] = strip_each_in(line[2].split(',')) + out[line[1]] = strip_each_in(line[2].split(",")) return out def _parse_summary(self): @@ -284,17 +290,17 @@ def _parse_summary(self): while True: summary = self._doc.read_to_next_empty_line() summary_str = " ".join([s.strip() for s in summary]).strip() - if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str): - self['Signature'] = summary_str + if re.compile("^([\w., ]+=)?\s*[\w\.]+\(.*\)$").match(summary_str): + self["Signature"] = summary_str if not self._is_at_section(): continue break if summary is not None: - self['Summary'] = summary + self["Summary"] = summary if not self._is_at_section(): - self['Extended Summary'] = self._read_to_next_section() + self["Extended Summary"] = self._read_to_next_section() def _parse(self): self._doc.reset() @@ -303,54 +309,61 @@ def _parse(self): sections = list(self._read_sections()) section_names = set([section for section, content in sections]) - has_returns = 'Returns' in section_names - has_yields = 'Yields' in section_names + has_returns = "Returns" in section_names + has_yields = "Yields" in section_names # We could do more tests, but we are not. Arbitrarily. if has_returns and has_yields: - msg = 'Docstring contains both a Returns and Yields section.' + msg = "Docstring contains both a Returns and Yields section." raise ValueError(msg) for (section, content) in sections: - if not section.startswith('..'): - section = (s.capitalize() for s in section.split(' ')) - section = ' '.join(section) - if section in ('Parameters', 'Returns', 'Yields', 'Raises', - 'Warns', 'Other Parameters', 'Attributes', - 'Methods'): + if not section.startswith(".."): + section = (s.capitalize() for s in section.split(" ")) + section = " ".join(section) + if section in ( + "Parameters", + "Returns", + "Yields", + "Raises", + "Warns", + "Other Parameters", + "Attributes", + "Methods", + ): self[section] = self._parse_param_list(content) - elif section.startswith('.. index::'): - self['index'] = self._parse_index(section, content) - elif section == 'See Also': - self['See Also'] = self._parse_see_also(content) + elif section.startswith(".. index::"): + self["index"] = self._parse_index(section, content) + elif section == "See Also": + self["See Also"] = self._parse_see_also(content) else: self[section] = content # string conversion routines - def _str_header(self, name, symbol='-'): - return [name, len(name)*symbol] + def _str_header(self, name, symbol="-"): + return [name, len(name) * symbol] def _str_indent(self, doc, indent=4): out = [] for line in doc: - out += [' '*indent + line] + out += [" " * indent + line] return out def _str_signature(self): - if self['Signature']: - return [self['Signature'].replace('*', '\*')] + [''] + if self["Signature"]: + return [self["Signature"].replace("*", "\*")] + [""] else: - return [''] + return [""] def _str_summary(self): - if self['Summary']: - return self['Summary'] + [''] + if self["Summary"]: + return self["Summary"] + [""] else: return [] def _str_extended_summary(self): - if self['Extended Summary']: - return self['Extended Summary'] + [''] + if self["Extended Summary"]: + return self["Extended Summary"] + [""] else: return [] @@ -360,11 +373,11 @@ def _str_param_list(self, name): out += self._str_header(name) for param, param_type, desc in self[name]: if param_type: - out += ['%s : %s' % (param, param_type)] + out += ["%s : %s" % (param, param_type)] else: out += [param] out += self._str_indent(desc) - out += [''] + out += [""] return out def _str_section(self, name): @@ -372,69 +385,75 @@ def _str_section(self, name): if self[name]: out += self._str_header(name) out += self[name] - out += [''] + out += [""] return out def _str_see_also(self, func_role): - if not self['See Also']: + if not self["See Also"]: return [] out = [] out += self._str_header("See Also") last_had_desc = True - for func, desc, role in self['See Also']: + for func, desc, role in self["See Also"]: if role: - link = ':%s:`%s`' % (role, func) + link = ":%s:`%s`" % (role, func) elif func_role: - link = ':%s:`%s`' % (func_role, func) + link = ":%s:`%s`" % (func_role, func) else: link = "`%s`_" % func if desc or last_had_desc: - out += [''] + out += [""] out += [link] else: out[-1] += ", %s" % link if desc: - out += self._str_indent([' '.join(desc)]) + out += self._str_indent([" ".join(desc)]) last_had_desc = True else: last_had_desc = False - out += [''] + out += [""] return out def _str_index(self): - idx = self['index'] + idx = self["index"] out = [] - out += ['.. index:: %s' % idx.get('default', '')] + out += [".. index:: %s" % idx.get("default", "")] for section, references in idx.items(): - if section == 'default': + if section == "default": continue - out += [' :%s: %s' % (section, ', '.join(references))] + out += [" :%s: %s" % (section, ", ".join(references))] return out - def __str__(self, func_role=''): + def __str__(self, func_role=""): out = [] out += self._str_signature() out += self._str_summary() out += self._str_extended_summary() - for param_list in ('Parameters', 'Returns', 'Yields', - 'Other Parameters', 'Raises', 'Warns'): + for param_list in ( + "Parameters", + "Returns", + "Yields", + "Other Parameters", + "Raises", + "Warns", + ): out += self._str_param_list(param_list) - out += self._str_section('Warnings') + out += self._str_section("Warnings") out += self._str_see_also(func_role) - for s in ('Notes', 'References', 'Examples'): + for s in ("Notes", "References", "Examples"): out += self._str_section(s) - for param_list in ('Attributes', 'Methods'): + for param_list in ("Attributes", "Methods"): out += self._str_param_list(param_list) out += self._str_index() - return '\n'.join(out) + return "\n".join(out) def indent(str, indent=4): - indent_str = ' '*indent + indent_str = " " * indent if str is None: return indent_str - lines = str.split('\n') - return '\n'.join(indent_str + l for l in lines) + lines = str.split("\n") + return "\n".join(indent_str + l for l in lines) def dedent_lines(lines): @@ -442,22 +461,22 @@ def dedent_lines(lines): return textwrap.dedent("\n".join(lines)).split("\n") -def header(text, style='-'): - return text + '\n' + style*len(text) + '\n' +def header(text, style="-"): + return text + "\n" + style * len(text) + "\n" class FunctionDoc(NumpyDocString): - def __init__(self, func, role='func', doc=None, config={}): + def __init__(self, func, role="func", doc=None, config={}): self._f = func self._role = role # e.g. "func" or "meth" if doc is None: if func is None: raise ValueError("No function or docstring given") - doc = inspect.getdoc(func) or '' + doc = inspect.getdoc(func) or "" NumpyDocString.__init__(self, doc) - if not self['Signature'] and func is not None: + if not self["Signature"] and func is not None: func, func_name = self.get_func() try: # try to read signature @@ -466,34 +485,32 @@ def __init__(self, func, role='func', doc=None, config={}): else: argspec = inspect.getargspec(func) argspec = inspect.formatargspec(*argspec) - argspec = argspec.replace('*', '\*') - signature = '%s%s' % (func_name, argspec) + argspec = argspec.replace("*", "\*") + signature = "%s%s" % (func_name, argspec) except TypeError as e: - signature = '%s()' % func_name - self['Signature'] = signature + signature = "%s()" % func_name + self["Signature"] = signature def get_func(self): - func_name = getattr(self._f, '__name__', self.__class__.__name__) + func_name = getattr(self._f, "__name__", self.__class__.__name__) if inspect.isclass(self._f): - func = getattr(self._f, '__call__', self._f.__init__) + func = getattr(self._f, "__call__", self._f.__init__) else: func = self._f return func, func_name def __str__(self): - out = '' + out = "" func, func_name = self.get_func() - signature = self['Signature'].replace('*', '\*') + signature = self["Signature"].replace("*", "\*") - roles = {'func': 'function', - 'meth': 'method'} + roles = {"func": "function", "meth": "method"} if self._role: if self._role not in roles: print("Warning: invalid role %s" % self._role) - out += '.. %s:: %s\n \n\n' % (roles.get(self._role, ''), - func_name) + out += ".. %s:: %s\n \n\n" % (roles.get(self._role, ""), func_name) out += super(FunctionDoc, self).__str__(func_role=self._role) return out @@ -501,19 +518,17 @@ def __str__(self): class ClassDoc(NumpyDocString): - extra_public_methods = ['__call__'] + extra_public_methods = ["__call__"] - def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc, - config={}): + def __init__(self, cls, doc=None, modulename="", func_doc=FunctionDoc, config={}): if not inspect.isclass(cls) and cls is not None: raise ValueError("Expected a class or None, but got %r" % cls) self._cls = cls - self.show_inherited_members = config.get( - 'show_inherited_class_members', True) + self.show_inherited_members = config.get("show_inherited_class_members", True) - if modulename and not modulename.endswith('.'): - modulename += '.' + if modulename and not modulename.endswith("."): + modulename += "." self._mod = modulename if doc is None: @@ -523,21 +538,24 @@ def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc, NumpyDocString.__init__(self, doc) - if config.get('show_class_members', True): + if config.get("show_class_members", True): + def splitlines_x(s): if not s: return [] else: return s.splitlines() - for field, items in [('Methods', self.methods), - ('Attributes', self.properties)]: + for field, items in [ + ("Methods", self.methods), + ("Attributes", self.properties), + ]: if not self[field]: doc_list = [] for name in sorted(items): try: doc_item = pydoc.getdoc(getattr(self._cls, name)) - doc_list.append((name, '', splitlines_x(doc_item))) + doc_list.append((name, "", splitlines_x(doc_item))) except AttributeError: pass # method doesn't exist self[field] = doc_list @@ -546,21 +564,33 @@ def splitlines_x(s): def methods(self): if self._cls is None: return [] - return [name for name, func in inspect.getmembers(self._cls) - if ((not name.startswith('_') - or name in self.extra_public_methods) - and isinstance(func, collections.Callable) - and self._is_show_member(name))] + return [ + name + for name, func in inspect.getmembers(self._cls) + if ( + (not name.startswith("_") or name in self.extra_public_methods) + and isinstance(func, collections.Callable) + and self._is_show_member(name) + ) + ] @property def properties(self): if self._cls is None: return [] - return [name for name, func in inspect.getmembers(self._cls) - if (not name.startswith('_') and - (func is None or isinstance(func, property) or - inspect.isgetsetdescriptor(func)) - and self._is_show_member(name))] + return [ + name + for name, func in inspect.getmembers(self._cls) + if ( + not name.startswith("_") + and ( + func is None + or isinstance(func, property) + or inspect.isgetsetdescriptor(func) + ) + and self._is_show_member(name) + ) + ] def _is_show_member(self, name): if self.show_inherited_members: diff --git a/docs/sphinxext/docscrape_sphinx.py b/docs/sphinxext/docscrape_sphinx.py index e44e770e..66ece8b9 100644 --- a/docs/sphinxext/docscrape_sphinx.py +++ b/docs/sphinxext/docscrape_sphinx.py @@ -2,55 +2,55 @@ import sphinx from docscrape import NumpyDocString, FunctionDoc, ClassDoc + class SphinxDocString(NumpyDocString): def __init__(self, docstring, config={}): - self.use_plots = config.get('use_plots', False) + self.use_plots = config.get("use_plots", False) NumpyDocString.__init__(self, docstring, config=config) # string conversion routines - def _str_header(self, name, symbol='`'): - return ['.. rubric:: ' + name, ''] + def _str_header(self, name, symbol="`"): + return [".. rubric:: " + name, ""] def _str_field_list(self, name): - return [':' + name + ':'] + return [":" + name + ":"] def _str_indent(self, doc, indent=4): out = [] for line in doc: - out += [' '*indent + line] + out += [" " * indent + line] return out def _str_signature(self): - return [''] - if self['Signature']: - return ['``%s``' % self['Signature']] + [''] + return [""] + if self["Signature"]: + return ["``%s``" % self["Signature"]] + [""] else: - return [''] + return [""] def _str_summary(self): - return self['Summary'] + [''] + return self["Summary"] + [""] def _str_extended_summary(self): - return self['Extended Summary'] + [''] + return self["Extended Summary"] + [""] def _str_param_list(self, name): out = [] if self[name]: out += self._str_field_list(name) - out += [''] - for param,param_type,desc in self[name]: - out += self._str_indent(['**%s** : %s' % (param.strip(), - param_type)]) - out += [''] - out += self._str_indent(desc,8) - out += [''] + out += [""] + for param, param_type, desc in self[name]: + out += self._str_indent(["**%s** : %s" % (param.strip(), param_type)]) + out += [""] + out += self._str_indent(desc, 8) + out += [""] return out @property def _obj(self): - if hasattr(self, '_cls'): + if hasattr(self, "_cls"): return self._cls - elif hasattr(self, '_f'): + elif hasattr(self, "_f"): return self._f return None @@ -62,11 +62,11 @@ def _str_member_list(self, name): """ out = [] if self[name]: - out += ['.. rubric:: %s' % name, ''] - prefix = getattr(self, '_name', '') + out += [".. rubric:: %s" % name, ""] + prefix = getattr(self, "_name", "") if prefix: - prefix = '~%s.' % prefix + prefix = "~%s." % prefix autosum = [] others = [] @@ -78,148 +78,159 @@ def _str_member_list(self, name): others.append((param, param_type, desc)) if autosum: - out += ['.. autosummary::', ' :toctree:', ''] + out += [".. autosummary::", " :toctree:", ""] out += autosum if others: maxlen_0 = max([len(x[0]) for x in others]) maxlen_1 = max([len(x[1]) for x in others]) - hdr = "="*maxlen_0 + " " + "="*maxlen_1 + " " + "="*10 - fmt = '%%%ds %%%ds ' % (maxlen_0, maxlen_1) + hdr = "=" * maxlen_0 + " " + "=" * maxlen_1 + " " + "=" * 10 + fmt = "%%%ds %%%ds " % (maxlen_0, maxlen_1) n_indent = maxlen_0 + maxlen_1 + 4 out += [hdr] for param, param_type, desc in others: out += [fmt % (param.strip(), param_type)] out += self._str_indent(desc, n_indent) out += [hdr] - out += [''] + out += [""] return out def _str_section(self, name): out = [] if self[name]: out += self._str_header(name) - out += [''] + out += [""] content = textwrap.dedent("\n".join(self[name])).split("\n") out += content - out += [''] + out += [""] return out def _str_see_also(self, func_role): out = [] - if self['See Also']: + if self["See Also"]: see_also = super(SphinxDocString, self)._str_see_also(func_role) - out = ['.. seealso::', ''] + out = [".. seealso::", ""] out += self._str_indent(see_also[2:]) return out def _str_warnings(self): out = [] - if self['Warnings']: - out = ['.. warning::', ''] - out += self._str_indent(self['Warnings']) + if self["Warnings"]: + out = [".. warning::", ""] + out += self._str_indent(self["Warnings"]) return out def _str_index(self): - idx = self['index'] + idx = self["index"] out = [] if len(idx) == 0: return out - out += ['.. index:: %s' % idx.get('default','')] + out += [".. index:: %s" % idx.get("default", "")] for section, references in idx.iteritems(): - if section == 'default': + if section == "default": continue - elif section == 'refguide': - out += [' single: %s' % (', '.join(references))] + elif section == "refguide": + out += [" single: %s" % (", ".join(references))] else: - out += [' %s: %s' % (section, ','.join(references))] + out += [" %s: %s" % (section, ",".join(references))] return out def _str_references(self): out = [] - if self['References']: - out += self._str_header('References') - if isinstance(self['References'], str): - self['References'] = [self['References']] - out.extend(self['References']) - out += [''] + if self["References"]: + out += self._str_header("References") + if isinstance(self["References"], str): + self["References"] = [self["References"]] + out.extend(self["References"]) + out += [""] # Latex collects all references to a separate bibliography, # so we need to insert links to it if sphinx.__version__ >= "0.6": - out += ['.. only:: latex',''] + out += [".. only:: latex", ""] else: - out += ['.. latexonly::',''] + out += [".. latexonly::", ""] items = [] - for line in self['References']: - m = re.match(r'.. \[([a-z0-9._-]+)\]', line, re.I) + for line in self["References"]: + m = re.match(r".. \[([a-z0-9._-]+)\]", line, re.I) if m: items.append(m.group(1)) - out += [' ' + ", ".join(["[%s]_" % item for item in items]), ''] + out += [" " + ", ".join(["[%s]_" % item for item in items]), ""] return out def _str_examples(self): - examples_str = "\n".join(self['Examples']) + examples_str = "\n".join(self["Examples"]) - if (self.use_plots and 'import matplotlib' in examples_str - and 'plot::' not in examples_str): + if ( + self.use_plots + and "import matplotlib" in examples_str + and "plot::" not in examples_str + ): out = [] - out += self._str_header('Examples') - out += ['.. plot::', ''] - out += self._str_indent(self['Examples']) - out += [''] + out += self._str_header("Examples") + out += [".. plot::", ""] + out += self._str_indent(self["Examples"]) + out += [""] return out else: - return self._str_section('Examples') + return self._str_section("Examples") def __str__(self, indent=0, func_role="obj"): out = [] out += self._str_signature() - out += self._str_index() + [''] + out += self._str_index() + [""] out += self._str_summary() out += self._str_extended_summary() - for param_list in ('Parameters', 'Returns', 'Other Parameters', - 'Raises', 'Warns'): + for param_list in ( + "Parameters", + "Returns", + "Other Parameters", + "Raises", + "Warns", + ): out += self._str_param_list(param_list) out += self._str_warnings() out += self._str_see_also(func_role) - out += self._str_section('Notes') + out += self._str_section("Notes") out += self._str_references() out += self._str_examples() - for param_list in ('Attributes', 'Methods'): + for param_list in ("Attributes", "Methods"): out += self._str_member_list(param_list) - out = self._str_indent(out,indent) - return '\n'.join(out) + out = self._str_indent(out, indent) + return "\n".join(out) + class SphinxFunctionDoc(SphinxDocString, FunctionDoc): def __init__(self, obj, doc=None, config={}): - self.use_plots = config.get('use_plots', False) + self.use_plots = config.get("use_plots", False) FunctionDoc.__init__(self, obj, doc=doc, config=config) + class SphinxClassDoc(SphinxDocString, ClassDoc): def __init__(self, obj, doc=None, func_doc=None, config={}): - self.use_plots = config.get('use_plots', False) + self.use_plots = config.get("use_plots", False) ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config) + class SphinxObjDoc(SphinxDocString): def __init__(self, obj, doc=None, config={}): self._f = obj SphinxDocString.__init__(self, doc, config=config) + def get_doc_object(obj, what=None, doc=None, config={}): if what is None: if inspect.isclass(obj): - what = 'class' + what = "class" elif inspect.ismodule(obj): - what = 'module' + what = "module" elif callable(obj): - what = 'function' + what = "function" else: - what = 'object' - if what == 'class': - return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc, - config=config) - elif what in ('function', 'method'): + what = "object" + if what == "class": + return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc, config=config) + elif what in ("function", "method"): return SphinxFunctionDoc(obj, doc=doc, config=config) else: if doc is None: diff --git a/docs/sphinxext/github.py b/docs/sphinxext/github.py index 519e146d..d9602ab8 100644 --- a/docs/sphinxext/github.py +++ b/docs/sphinxext/github.py @@ -20,6 +20,7 @@ from docutils import nodes, utils from docutils.parsers.rst.roles import set_classes + def make_link_node(rawtext, app, type, slug, options): """Create a link to a github resource. @@ -34,20 +35,24 @@ def make_link_node(rawtext, app, type, slug, options): base = app.config.github_project_url if not base: raise AttributeError - if not base.endswith('/'): - base += '/' + if not base.endswith("/"): + base += "/" except AttributeError as err: - raise ValueError('github_project_url configuration value is not set (%s)' % str(err)) + raise ValueError( + "github_project_url configuration value is not set (%s)" % str(err) + ) - ref = base + type + '/' + slug + '/' + ref = base + type + "/" + slug + "/" set_classes(options) prefix = "#" - if type == 'pull': + if type == "pull": prefix = "PR " + prefix - node = nodes.reference(rawtext, prefix + utils.unescape(slug), refuri=ref, - **options) + node = nodes.reference( + rawtext, prefix + utils.unescape(slug), refuri=ref, **options + ) return node + def ghissue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): """Link to a GitHub issue. @@ -70,25 +75,29 @@ def ghissue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): raise ValueError except ValueError: msg = inliner.reporter.error( - 'GitHub issue number must be a number greater than or equal to 1; ' - '"%s" is invalid.' % text, line=lineno) + "GitHub issue number must be a number greater than or equal to 1; " + '"%s" is invalid.' % text, + line=lineno, + ) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] app = inliner.document.settings.env.app - #app.info('issue %r' % text) - if 'pull' in name.lower(): - category = 'pull' - elif 'issue' in name.lower(): - category = 'issues' + # app.info('issue %r' % text) + if "pull" in name.lower(): + category = "pull" + elif "issue" in name.lower(): + category = "issues" else: msg = inliner.reporter.error( - 'GitHub roles include "ghpull" and "ghissue", ' - '"%s" is invalid.' % name, line=lineno) + 'GitHub roles include "ghpull" and "ghissue", ' '"%s" is invalid.' % name, + line=lineno, + ) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] node = make_link_node(rawtext, app, category, str(issue_num), options) return [node], [] + def ghuser_role(name, rawtext, text, lineno, inliner, options={}, content=[]): """Link to a GitHub user. @@ -105,11 +114,12 @@ def ghuser_role(name, rawtext, text, lineno, inliner, options={}, content=[]): :param content: The directive content for customization. """ app = inliner.document.settings.env.app - #app.info('user link %r' % text) - ref = 'https://www.github.com/' + text + # app.info('user link %r' % text) + ref = "https://www.github.com/" + text node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] + def ghcommit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): """Link to a GitHub commit. @@ -126,15 +136,17 @@ def ghcommit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): :param content: The directive content for customization. """ app = inliner.document.settings.env.app - #app.info('user link %r' % text) + # app.info('user link %r' % text) try: base = app.config.github_project_url if not base: raise AttributeError - if not base.endswith('/'): - base += '/' + if not base.endswith("/"): + base += "/" except AttributeError as err: - raise ValueError('github_project_url configuration value is not set (%s)' % str(err)) + raise ValueError( + "github_project_url configuration value is not set (%s)" % str(err) + ) ref = base + text node = nodes.reference(rawtext, text[:6], refuri=ref, **options) @@ -146,10 +158,10 @@ def setup(app): :param app: Sphinx application context. """ - app.info('Initializing GitHub plugin') - app.add_role('ghissue', ghissue_role) - app.add_role('ghpull', ghissue_role) - app.add_role('ghuser', ghuser_role) - app.add_role('ghcommit', ghcommit_role) - app.add_config_value('github_project_url', None, 'env') + app.info("Initializing GitHub plugin") + app.add_role("ghissue", ghissue_role) + app.add_role("ghpull", ghissue_role) + app.add_role("ghuser", ghuser_role) + app.add_role("ghcommit", ghcommit_role) + app.add_config_value("github_project_url", None, "env") return diff --git a/docs/sphinxext/math_dollar.py b/docs/sphinxext/math_dollar.py index ad415deb..ba5af635 100644 --- a/docs/sphinxext/math_dollar.py +++ b/docs/sphinxext/math_dollar.py @@ -1,5 +1,6 @@ import re + def dollars_to_math(source): r""" Replace dollar signs with backticks. @@ -30,12 +31,14 @@ def dollars_to_math(source): # string, and later on we substitute the original back. global _data _data = {} + def repl(matchobj): global _data s = matchobj.group(0) t = "___XXX_REPL_%d___" % len(_data) _data[t] = s return t + s = re.sub(r"({[^{}$]*\$[^{}$]*\$[^{}]*})", repl, s) # matches $...$ dollars = re.compile(r"(?= 3: sixu = lambda s: s else: - sixu = lambda s: unicode(s, 'unicode_escape') + sixu = lambda s: unicode(s, "unicode_escape") -def mangle_docstrings(app, what, name, obj, options, lines, - reference_offset=[0]): +def mangle_docstrings(app, what, name, obj, options, lines, reference_offset=[0]): - cfg = {'use_plots': app.config.numpydoc_use_plots, - 'show_class_members': app.config.numpydoc_show_class_members, - 'show_inherited_class_members': - app.config.numpydoc_show_inherited_class_members, - 'class_members_toctree': app.config.numpydoc_class_members_toctree} + cfg = { + "use_plots": app.config.numpydoc_use_plots, + "show_class_members": app.config.numpydoc_show_class_members, + "show_inherited_class_members": app.config.numpydoc_show_inherited_class_members, + "class_members_toctree": app.config.numpydoc_class_members_toctree, + } - u_NL = sixu('\n') - if what == 'module': + u_NL = sixu("\n") + if what == "module": # Strip top title - pattern = '^\\s*[#*=]{4,}\\n[a-z0-9 -]+\\n[#*=]{4,}\\s*' + pattern = "^\\s*[#*=]{4,}\\n[a-z0-9 -]+\\n[#*=]{4,}\\s*" title_re = re.compile(sixu(pattern), re.I | re.S) - lines[:] = title_re.sub(sixu(''), u_NL.join(lines)).split(u_NL) + lines[:] = title_re.sub(sixu(""), u_NL.join(lines)).split(u_NL) else: doc = get_doc_object(obj, what, u_NL.join(lines), config=cfg) if sys.version_info[0] >= 3: @@ -60,21 +60,21 @@ def mangle_docstrings(app, what, name, obj, options, lines, doc = unicode(doc) lines[:] = doc.split(u_NL) - if (app.config.numpydoc_edit_link and hasattr(obj, '__name__') and - obj.__name__): - if hasattr(obj, '__module__'): + if app.config.numpydoc_edit_link and hasattr(obj, "__name__") and obj.__name__: + if hasattr(obj, "__module__"): v = dict(full_name=sixu("%s.%s") % (obj.__module__, obj.__name__)) else: v = dict(full_name=obj.__name__) - lines += [sixu(''), sixu('.. htmlonly::'), sixu('')] - lines += [sixu(' %s') % x for x in - (app.config.numpydoc_edit_link % v).split("\n")] + lines += [sixu(""), sixu(".. htmlonly::"), sixu("")] + lines += [ + sixu(" %s") % x for x in (app.config.numpydoc_edit_link % v).split("\n") + ] # replace reference numbers so that there are no duplicates references = [] for line in lines: line = line.strip() - m = re.match(sixu('^.. \\[([a-z0-9_.-])\\]'), line, re.I) + m = re.match(sixu("^.. \\[([a-z0-9_.-])\\]"), line, re.I) if m: references.append(m.group(1)) @@ -83,57 +83,60 @@ def mangle_docstrings(app, what, name, obj, options, lines, if references: for i, line in enumerate(lines): for r in references: - if re.match(sixu('^\\d+$'), r): + if re.match(sixu("^\\d+$"), r): new_r = sixu("R%d") % (reference_offset[0] + int(r)) else: new_r = sixu("%s%d") % (r, reference_offset[0]) - lines[i] = lines[i].replace(sixu('[%s]_') % r, - sixu('[%s]_') % new_r) - lines[i] = lines[i].replace(sixu('.. [%s]') % r, - sixu('.. [%s]') % new_r) + lines[i] = lines[i].replace(sixu("[%s]_") % r, sixu("[%s]_") % new_r) + lines[i] = lines[i].replace( + sixu(".. [%s]") % r, sixu(".. [%s]") % new_r + ) reference_offset[0] += len(references) def mangle_signature(app, what, name, obj, options, sig, retann): # Do not try to inspect classes that don't define `__init__` - if (inspect.isclass(obj) and - (not hasattr(obj, '__init__') or - 'initializes x; see ' in pydoc.getdoc(obj.__init__))): - return '', '' - - if not (isinstance(obj, collections.Callable) or - hasattr(obj, '__argspec_is_invalid_')): + if inspect.isclass(obj) and ( + not hasattr(obj, "__init__") + or "initializes x; see " in pydoc.getdoc(obj.__init__) + ): + return "", "" + + if not ( + isinstance(obj, collections.Callable) or hasattr(obj, "__argspec_is_invalid_") + ): return - if not hasattr(obj, '__doc__'): + if not hasattr(obj, "__doc__"): return doc = SphinxDocString(pydoc.getdoc(obj)) - if doc['Signature']: - sig = re.sub(sixu("^[^(]*"), sixu(""), doc['Signature']) - return sig, sixu('') + if doc["Signature"]: + sig = re.sub(sixu("^[^(]*"), sixu(""), doc["Signature"]) + return sig, sixu("") def setup(app, get_doc_object_=get_doc_object): - if not hasattr(app, 'add_config_value'): + if not hasattr(app, "add_config_value"): return # probably called by nose, better bail out global get_doc_object get_doc_object = get_doc_object_ - app.connect('autodoc-process-docstring', mangle_docstrings) - app.connect('autodoc-process-signature', mangle_signature) - app.add_config_value('numpydoc_edit_link', None, False) - app.add_config_value('numpydoc_use_plots', None, False) - app.add_config_value('numpydoc_show_class_members', True, True) - app.add_config_value('numpydoc_show_inherited_class_members', True, True) - app.add_config_value('numpydoc_class_members_toctree', True, True) + app.connect("autodoc-process-docstring", mangle_docstrings) + app.connect("autodoc-process-signature", mangle_signature) + app.add_config_value("numpydoc_edit_link", None, False) + app.add_config_value("numpydoc_use_plots", None, False) + app.add_config_value("numpydoc_show_class_members", True, True) + app.add_config_value("numpydoc_show_inherited_class_members", True, True) + app.add_config_value("numpydoc_class_members_toctree", True, True) # Extra mangling domains app.add_domain(NumpyPythonDomain) app.add_domain(NumpyCDomain) + # ------------------------------------------------------------------------------ # Docstring-mangling domains # ------------------------------------------------------------------------------ @@ -153,31 +156,32 @@ def __init__(self, *a, **kw): def wrap_mangling_directives(self): for name, objtype in list(self.directive_mangling_map.items()): self.directives[name] = wrap_mangling_directive( - self.directives[name], objtype) + self.directives[name], objtype + ) class NumpyPythonDomain(ManglingDomainBase, PythonDomain): - name = 'np' + name = "np" directive_mangling_map = { - 'function': 'function', - 'class': 'class', - 'exception': 'class', - 'method': 'function', - 'classmethod': 'function', - 'staticmethod': 'function', - 'attribute': 'attribute', + "function": "function", + "class": "class", + "exception": "class", + "method": "function", + "classmethod": "function", + "staticmethod": "function", + "attribute": "attribute", } indices = [] class NumpyCDomain(ManglingDomainBase, CDomain): - name = 'np-c' + name = "np-c" directive_mangling_map = { - 'function': 'function', - 'member': 'attribute', - 'macro': 'function', - 'type': 'class', - 'var': 'object', + "function": "function", + "member": "attribute", + "macro": "function", + "type": "class", + "var": "object", } @@ -188,7 +192,7 @@ def run(self): name = None if self.arguments: - m = re.match(r'^(.*\s+)?(.*?)(\(.*)?', self.arguments[0]) + m = re.match(r"^(.*\s+)?(.*?)(\(.*)?", self.arguments[0]) name = m.group(2).strip() if not name: diff --git a/docs/tools/apigen.py b/docs/tools/apigen.py index 758671fa..716fd488 100644 --- a/docs/tools/apigen.py +++ b/docs/tools/apigen.py @@ -27,21 +27,23 @@ # suppress print statements (warnings for empty files) DEBUG = True + class ApiDocWriter(object): - ''' Class for automatic detection and parsing of API docs - to Sphinx-parsable reST format''' + """ Class for automatic detection and parsing of API docs + to Sphinx-parsable reST format""" # only separating first two levels - rst_section_levels = ['*', '=', '-', '~', '^'] - - def __init__(self, - package_name, - rst_extension='.txt', - package_skip_patterns=None, - module_skip_patterns=None, - other_defines = True - ): - ''' Initialize package for parsing + rst_section_levels = ["*", "=", "-", "~", "^"] + + def __init__( + self, + package_name, + rst_extension=".txt", + package_skip_patterns=None, + module_skip_patterns=None, + other_defines=True, + ): + """ Initialize package for parsing Parameters ---------- @@ -69,11 +71,11 @@ def __init__(self, other_defines : {True, False}, optional Whether to include classes and functions that are imported in a particular module but not defined there. - ''' + """ if package_skip_patterns is None: - package_skip_patterns = ['\\.tests$'] + package_skip_patterns = ["\\.tests$"] if module_skip_patterns is None: - module_skip_patterns = ['\\.setup$', '\\._'] + module_skip_patterns = ["\\.setup$", "\\._"] self.package_name = package_name self.rst_extension = rst_extension self.package_skip_patterns = package_skip_patterns @@ -84,7 +86,7 @@ def get_package_name(self): return self._package_name def set_package_name(self, package_name): - ''' Set package_name + """ Set package_name >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx @@ -94,26 +96,27 @@ def set_package_name(self, package_name): >>> import docutils >>> docwriter.root_path == docutils.__path__[0] True - ''' + """ # It's also possible to imagine caching the module parsing here self._package_name = package_name root_module = self._import(package_name) self.root_path = root_module.__path__[-1] self.written_modules = None - package_name = property(get_package_name, set_package_name, None, - 'get/set package_name') + package_name = property( + get_package_name, set_package_name, None, "get/set package_name" + ) def _import(self, name): - ''' Import namespace package ''' + """ Import namespace package """ mod = __import__(name) - components = name.split('.') + components = name.split(".") for comp in components[1:]: mod = getattr(mod, comp) return mod def _get_object_name(self, line): - ''' Get second token in line + """ Get second token in line >>> docwriter = ApiDocWriter('sphinx') >>> docwriter._get_object_name(" def func(): ") 'func' @@ -121,14 +124,14 @@ def _get_object_name(self, line): 'Klass' >>> docwriter._get_object_name(" class Klass: ") 'Klass' - ''' - name = line.split()[1].split('(')[0].strip() + """ + name = line.split()[1].split("(")[0].strip() # in case we have classes which are not derived from object # ie. old style classes - return name.rstrip(':') + return name.rstrip(":") def _uri2path(self, uri): - ''' Convert uri to absolute filepath + """ Convert uri to absolute filepath Parameters ---------- @@ -154,38 +157,38 @@ def _uri2path(self, uri): True >>> docwriter._uri2path('sphinx.does_not_exist') - ''' + """ if uri == self.package_name: - return os.path.join(self.root_path, '__init__.py') - path = uri.replace(self.package_name + '.', '') - path = path.replace('.', os.path.sep) + return os.path.join(self.root_path, "__init__.py") + path = uri.replace(self.package_name + ".", "") + path = path.replace(".", os.path.sep) path = os.path.join(self.root_path, path) # XXX maybe check for extensions as well? - if os.path.exists(path + '.py'): # file - path += '.py' - elif os.path.exists(os.path.join(path, '__init__.py')): - path = os.path.join(path, '__init__.py') + if os.path.exists(path + ".py"): # file + path += ".py" + elif os.path.exists(os.path.join(path, "__init__.py")): + path = os.path.join(path, "__init__.py") else: return None return path def _path2uri(self, dirpath): - ''' Convert directory path to uri ''' - package_dir = self.package_name.replace('.', os.path.sep) + """ Convert directory path to uri """ + package_dir = self.package_name.replace(".", os.path.sep) relpath = dirpath.replace(self.root_path, package_dir) if relpath.startswith(os.path.sep): relpath = relpath[1:] - return relpath.replace(os.path.sep, '.') + return relpath.replace(os.path.sep, ".") def _parse_module(self, uri): - ''' Parse module defined in *uri* ''' + """ Parse module defined in *uri* """ filename = self._uri2path(uri) if filename is None: - print(filename, 'erk') + print(filename, "erk") # nothing that we could handle here. - return ([],[]) + return ([], []) - f = open(filename, 'rt') + f = open(filename, "rt") functions, classes = self._parse_lines(f) f.close() return functions, classes @@ -208,7 +211,7 @@ def _parse_module_with_import(self, uri): """ mod = __import__(uri, fromlist=[uri]) # find all public objects in the module. - obj_strs = [obj for obj in dir(mod) if not obj.startswith('_')] + obj_strs = [obj for obj in dir(mod) if not obj.startswith("_")] functions = [] classes = [] for obj_str in obj_strs: @@ -220,9 +223,11 @@ def _parse_module_with_import(self, uri): if not self.other_defines and not getmodule(obj) == mod: continue # figure out if obj is a function or class - if hasattr(obj, 'func_name') or \ - isinstance(obj, BuiltinFunctionType) or \ - isinstance(obj, FunctionType): + if ( + hasattr(obj, "func_name") + or isinstance(obj, BuiltinFunctionType) + or isinstance(obj, FunctionType) + ): functions.append(obj_str) else: try: @@ -234,19 +239,19 @@ def _parse_module_with_import(self, uri): return functions, classes def _parse_lines(self, linesource): - ''' Parse lines of text for functions and classes ''' + """ Parse lines of text for functions and classes """ functions = [] classes = [] for line in linesource: - if line.startswith('def ') and line.count('('): + if line.startswith("def ") and line.count("("): # exclude private stuff name = self._get_object_name(line) - if not name.startswith('_'): + if not name.startswith("_"): functions.append(name) - elif line.startswith('class '): + elif line.startswith("class "): # exclude private stuff name = self._get_object_name(line) - if not name.startswith('_'): + if not name.startswith("_"): classes.append(name) else: pass @@ -255,7 +260,7 @@ def _parse_lines(self, linesource): return functions, classes def generate_api_doc(self, uri): - '''Make autodoc documentation template string for a module + """Make autodoc documentation template string for a module Parameters ---------- @@ -268,57 +273,63 @@ def generate_api_doc(self, uri): Module name, table of contents. body : string Function and class docstrings. - ''' + """ # get the names of all classes and functions functions, classes = self._parse_module_with_import(uri) if not len(functions) and not len(classes) and DEBUG: - print('WARNING: Empty -', uri) # dbg + print("WARNING: Empty -", uri) # dbg # Make a shorter version of the uri that omits the package name for # titles - uri_short = re.sub(r'^%s\.' % self.package_name,'',uri) + uri_short = re.sub(r"^%s\." % self.package_name, "", uri) - head = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' - body = '' + head = ".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n" + body = "" # Set the chapter title to read 'module' for all modules except for the # main packages - if '.' in uri_short: - title = 'Module: :mod:`' + uri_short + '`' - head += title + '\n' + self.rst_section_levels[2] * len(title) + if "." in uri_short: + title = "Module: :mod:`" + uri_short + "`" + head += title + "\n" + self.rst_section_levels[2] * len(title) else: - title = ':mod:`' + uri_short + '`' - head += title + '\n' + self.rst_section_levels[1] * len(title) + title = ":mod:`" + uri_short + "`" + head += title + "\n" + self.rst_section_levels[1] * len(title) - head += '\n.. automodule:: ' + uri + '\n' - head += '\n.. currentmodule:: ' + uri + '\n' - body += '\n.. currentmodule:: ' + uri + '\n\n' + head += "\n.. automodule:: " + uri + "\n" + head += "\n.. currentmodule:: " + uri + "\n" + body += "\n.. currentmodule:: " + uri + "\n\n" for c in classes: - body += '\n:class:`' + c + '`\n' \ - + self.rst_section_levels[3] * \ - (len(c)+9) + '\n\n' - body += '\n.. autoclass:: ' + c + '\n' + body += ( + "\n:class:`" + + c + + "`\n" + + self.rst_section_levels[3] * (len(c) + 9) + + "\n\n" + ) + body += "\n.. autoclass:: " + c + "\n" # must NOT exclude from index to keep cross-refs working - body += ' :members:\n' \ - ' :undoc-members:\n' \ - ' :show-inheritance:\n' \ - '\n' \ - ' .. automethod:: __init__\n\n' - head += '.. autosummary::\n\n' + body += ( + " :members:\n" + " :undoc-members:\n" + " :show-inheritance:\n" + "\n" + " .. automethod:: __init__\n\n" + ) + head += ".. autosummary::\n\n" for f in classes + functions: - head += ' ' + f + '\n' - head += '\n' + head += " " + f + "\n" + head += "\n" for f in functions: # must NOT exclude from index to keep cross-refs working - body += f + '\n' - body += self.rst_section_levels[3] * len(f) + '\n' - body += '\n.. autofunction:: ' + f + '\n\n' + body += f + "\n" + body += self.rst_section_levels[3] * len(f) + "\n" + body += "\n.. autofunction:: " + f + "\n\n" return head, body def _survives_exclude(self, matchstr, match_type): - ''' Returns True if *matchstr* does not match patterns + """ Returns True if *matchstr* does not match patterns ``self.package_name`` removed from front of string if present @@ -337,14 +348,13 @@ def _survives_exclude(self, matchstr, match_type): >>> dw.module_skip_patterns.append('^\\.badmod$') >>> dw._survives_exclude('sphinx.badmod', 'module') False - ''' - if match_type == 'module': + """ + if match_type == "module": patterns = self.module_skip_patterns - elif match_type == 'package': + elif match_type == "package": patterns = self.package_skip_patterns else: - raise ValueError('Cannot interpret match type "%s"' - % match_type) + raise ValueError('Cannot interpret match type "%s"' % match_type) # Match to URI without package name L = len(self.package_name) if matchstr[:L] == self.package_name: @@ -360,7 +370,7 @@ def _survives_exclude(self, matchstr, match_type): return True def discover_modules(self): - ''' Return module sequence discovered from ``self.package_name`` + """ Return module sequence discovered from ``self.package_name`` Parameters @@ -382,38 +392,42 @@ def discover_modules(self): >>> 'sphinx.util' in dw.discover_modules() False >>> - ''' + """ modules = [self.package_name] # raw directory parsing for dirpath, dirnames, filenames in os.walk(self.root_path): # Check directory names for packages - root_uri = self._path2uri(os.path.join(self.root_path, - dirpath)) + root_uri = self._path2uri(os.path.join(self.root_path, dirpath)) # Normally, we'd only iterate over dirnames, but since # dipy does not import a whole bunch of modules we'll # include those here as well (the *.py filenames). - filenames = [f[:-3] for f in filenames if - f.endswith('.py') and not f.startswith('__init__')] + filenames = [ + f[:-3] + for f in filenames + if f.endswith(".py") and not f.startswith("__init__") + ] for filename in filenames: - package_uri = '/'.join((dirpath, filename)) + package_uri = "/".join((dirpath, filename)) for subpkg_name in dirnames + filenames: - package_uri = '.'.join((root_uri, subpkg_name)) + package_uri = ".".join((root_uri, subpkg_name)) package_path = self._uri2path(package_uri) - if (package_path and - self._survives_exclude(package_uri, 'package')): + if package_path and self._survives_exclude(package_uri, "package"): modules.append(package_uri) return sorted(modules) def write_modules_api(self, modules, outdir): # upper-level modules - main_module = modules[0].split('.')[0] - ulms = ['.'.join(m.split('.')[:2]) if m.count('.') >= 1 - else m.split('.')[0] for m in modules] + main_module = modules[0].split(".")[0] + ulms = [ + ".".join(m.split(".")[:2]) if m.count(".") >= 1 else m.split(".")[0] + for m in modules + ] from collections import OrderedDict + module_by_ulm = OrderedDict() for v, k in zip(modules, ulms): @@ -438,7 +452,7 @@ def write_modules_api(self, modules, outdir): out_module = ulm + self.rst_extension outfile = os.path.join(outdir, out_module) - fileobj = open(outfile, 'wt') + fileobj = open(outfile, "wt") fileobj.writelines(document_head + document_body) fileobj.close() @@ -467,9 +481,9 @@ def write_api_docs(self, outdir): os.mkdir(outdir) # compose list of modules modules = self.discover_modules() - self.write_modules_api(modules,outdir) + self.write_modules_api(modules, outdir) - def write_index(self, outdir, froot='gen', relative_to=None): + def write_index(self, outdir, froot="gen", relative_to=None): """Make a reST API index file from written files Parameters @@ -488,22 +502,22 @@ def write_index(self, outdir, froot='gen', relative_to=None): leave path as it is. """ if self.written_modules is None: - raise ValueError('No modules written') + raise ValueError("No modules written") # Get full filename path - path = os.path.join(outdir, froot+self.rst_extension) + path = os.path.join(outdir, froot + self.rst_extension) # Path written into index is relative to rootpath if relative_to is not None: - relpath = (outdir + os.path.sep).replace(relative_to + os.path.sep, '') + relpath = (outdir + os.path.sep).replace(relative_to + os.path.sep, "") else: relpath = outdir - idx = open(path,'wt') + idx = open(path, "wt") w = idx.write - w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') + w(".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n") title = "API Reference" w(title + "\n") w("=" * len(title) + "\n\n") - w('.. toctree::\n\n') + w(".. toctree::\n\n") for f in self.written_modules: - w(' %s\n' % os.path.join(relpath,f)) + w(" %s\n" % os.path.join(relpath, f)) idx.close() diff --git a/docs/tools/buildmodref.py b/docs/tools/buildmodref.py index e2a7b9f6..769c696b 100755 --- a/docs/tools/buildmodref.py +++ b/docs/tools/buildmodref.py @@ -13,10 +13,11 @@ # version comparison from distutils.version import LooseVersion as V -#***************************************************************************** +# ***************************************************************************** + def abort(error): - print('*WARNING* API documentation not generated: %s' % error) + print("*WARNING* API documentation not generated: %s" % error) exit() @@ -40,18 +41,19 @@ def writeapi(package, outdir, source_version, other_defines=True): if source_version != installed_version: abort("Installed version does not match source version") - docwriter = ApiDocWriter(package, rst_extension='.rst', - other_defines=other_defines) + docwriter = ApiDocWriter(package, rst_extension=".rst", other_defines=other_defines) - docwriter.package_skip_patterns += [r'\.%s$' % package, - r'.*test.*$', - r'\.version.*$'] + docwriter.package_skip_patterns += [ + r"\.%s$" % package, + r".*test.*$", + r"\.version.*$", + ] docwriter.write_api_docs(outdir) - docwriter.write_index(outdir, 'index', relative_to=outdir) - print('%d files written' % len(docwriter.written_modules)) + docwriter.write_index(outdir, "index", relative_to=outdir) + print("%d files written" % len(docwriter.written_modules)) -if __name__ == '__main__': +if __name__ == "__main__": package = sys.argv[1] outdir = sys.argv[2] try: @@ -59,6 +61,6 @@ def writeapi(package, outdir, source_version, other_defines=True): except IndexError: other_defines = True else: - other_defines = other_defines in ('True', 'true', '1') + other_defines = other_defines in ("True", "true", "1") writeapi(package, outdir, other_defines=other_defines) diff --git a/get_version.py b/get_version.py index 7df90d44..141f8717 100644 --- a/get_version.py +++ b/get_version.py @@ -6,10 +6,11 @@ def main(): - sys.path.insert(0, op.abspath('.')) + sys.path.insert(0, op.abspath(".")) from templateflow.__about__ import __version__ + print(__version__) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/setup.cfg b/setup.cfg index d1df7d1e..b58c5f02 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,11 +31,6 @@ test_requires = packages = find: include_package_data = True -[options.package_data] -templateflow = - conf/config.json - conf/templateflow-skel.zip - [options.packages.find] exclude = *.tests diff --git a/setup.py b/setup.py index bb5b9966..551dabab 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,9 @@ # Give setuptools a hint to complain if it's too old a version # 30.3.0 allows us to put most metadata in setup.cfg # Should match pyproject.toml -SETUP_REQUIRES = ['setuptools >= 30.3.0'] +SETUP_REQUIRES = ["setuptools >= 30.3.0"] # This enables setuptools to install wheel on-the-fly -SETUP_REQUIRES += ['wheel'] if 'bdist_wheel' in sys.argv else [] +SETUP_REQUIRES += ["wheel"] if "bdist_wheel" in sys.argv else [] def make_cmdclass(basecmd): @@ -19,8 +19,9 @@ def make_cmdclass(basecmd): base_run = basecmd.run def new_run(self): - from templateflow.conf import update_home - update_home() + from templateflow.conf import setup_home + + setup_home() base_run(self) basecmd.run = new_run @@ -31,19 +32,19 @@ def new_run(self): class CheckHomeDevCommand(develop): pass + @make_cmdclass class CheckHomeProdCommand(install): pass -if __name__ == '__main__': +if __name__ == "__main__": """ Install entry-point """ setup( - name='templateflow', + name="templateflow", version=versioneer.get_version(), setup_requires=SETUP_REQUIRES, - cmdclass=versioneer.get_cmdclass(cmdclass={ - 'develop': CheckHomeDevCommand, - 'install': CheckHomeProdCommand, - }), + cmdclass=versioneer.get_cmdclass( + cmdclass={"develop": CheckHomeDevCommand, "install": CheckHomeProdCommand,} + ), ) diff --git a/templateflow/__about__.py b/templateflow/__about__.py index 7f774392..8115d7d2 100644 --- a/templateflow/__about__.py +++ b/templateflow/__about__.py @@ -2,9 +2,18 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """Versioning and other metadata about TemplateFlow.""" from ._version import get_versions -__version__ = get_versions()['version'] + +__version__ = get_versions()["version"] del get_versions -__copyright__ = 'Copyright 2019, Center for Reproducible Neuroscience, Stanford University' -__credits__ = ['Oscar Esteban', 'Chris Markiewicz', 'Rastko Ciric', 'Yaroslav O. Halchenko', - 'Chris Gorgolewski', 'Russell Poldrack'] +__copyright__ = ( + "Copyright 2019, Center for Reproducible Neuroscience, Stanford University" +) +__credits__ = [ + "Oscar Esteban", + "Chris Markiewicz", + "Rastko Ciric", + "Yaroslav O. Halchenko", + "Chris Gorgolewski", + "Russell Poldrack", +] diff --git a/templateflow/__init__.py b/templateflow/__init__.py index 09730638..500addc8 100644 --- a/templateflow/__init__.py +++ b/templateflow/__init__.py @@ -3,10 +3,10 @@ """TemplateFlow is the Zone of Templates.""" from .__about__ import __version__, __copyright__, __credits__ -__packagename__ = 'templateflow' +__packagename__ = "templateflow" __all__ = [ - '__copyright__', - '__credits__', - '__packagename__', - '__version__', + "__copyright__", + "__credits__", + "__packagename__", + "__version__", ] diff --git a/templateflow/_version.py b/templateflow/_version.py index 41475a3d..dc3be923 100644 --- a/templateflow/_version.py +++ b/templateflow/_version.py @@ -1,4 +1,3 @@ - # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -58,17 +57,18 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -76,10 +76,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -116,16 +119,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -190,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -201,16 +210,23 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -225,8 +241,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -234,10 +249,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -260,17 +284,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -279,8 +302,10 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] @@ -293,13 +318,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -330,8 +355,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -445,11 +469,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -469,9 +495,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } def get_versions(): @@ -485,8 +515,7 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass @@ -495,13 +524,16 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for i in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None, + } try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -515,6 +547,10 @@ def get_versions(): except NotThisMethod: pass - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } diff --git a/templateflow/api.py b/templateflow/api.py index 964278e9..5a65940a 100644 --- a/templateflow/api.py +++ b/templateflow/api.py @@ -1,6 +1,4 @@ -""" -TemplateFlow's Python Client -""" +"""TemplateFlow's Python Client.""" from json import loads from pathlib import Path import re @@ -53,8 +51,9 @@ def get(template, **kwargs): '.../tpl-fsLR_hemi-L_den-32k_sphere.surf.gii' """ - out_file = [Path(p) for p in TF_LAYOUT.get( - template=template, return_type='file', **kwargs)] + out_file = [ + Path(p) for p in TF_LAYOUT.get(template=template, return_type="file", **kwargs) + ] # Try DataLad first dl_missing = [p for p in out_file if not p.is_file()] @@ -64,29 +63,33 @@ def get(template, **kwargs): dl_missing.remove(filepath) # Fall-back to S3 if some files are still missing - s3_missing = [p for p in out_file - if p.is_file() and p.stat().st_size == 0] + s3_missing = [p for p in out_file if p.is_file() and p.stat().st_size == 0] for filepath in s3_missing + dl_missing: _s3_get(filepath) - not_fetched = [str(p) for p in out_file - if not p.is_file() or p.stat().st_size == 0] + not_fetched = [str(p) for p in out_file if not p.is_file() or p.stat().st_size == 0] if not_fetched: - msg = "Could not fetch template files: %s." % ', '.join(not_fetched) + msg = "Could not fetch template files: %s." % ", ".join(not_fetched) if dl_missing and not TF_USE_DATALAD: - msg += """\ + msg += ( + """\ The $TEMPLATEFLOW_HOME folder %s seems to contain an initiated DataLad \ dataset, but the environment variable $TEMPLATEFLOW_USE_DATALAD is not \ set or set to one of (false, off, 0). Please set $TEMPLATEFLOW_USE_DATALAD \ -on (possible values: true, on, 1).""" % TF_LAYOUT.root +on (possible values: true, on, 1).""" + % TF_LAYOUT.root + ) if s3_missing and TF_USE_DATALAD: - msg += """\ + msg += ( + """\ The $TEMPLATEFLOW_HOME folder %s seems to contain an plain \ dataset, but the environment variable $TEMPLATEFLOW_USE_DATALAD is \ set to one of (true, on, 1). Please set $TEMPLATEFLOW_USE_DATALAD \ -off (possible values: false, off, 0).""" % TF_LAYOUT.root +off (possible values: false, off, 0).""" + % TF_LAYOUT.root + ) raise RuntimeError(msg) @@ -141,7 +144,7 @@ def get_metadata(template): """ tf_home = Path(TF_LAYOUT.root) - filepath = tf_home / ('tpl-%s' % template) / 'template_description.json' + filepath = tf_home / ("tpl-%s" % template) / "template_description.json" # Ensure that template is installed and file is available if not filepath.is_file(): @@ -162,7 +165,7 @@ def get_citations(template, bibtex=False): """ data = get_metadata(template) - refs = data.get('ReferencesAndLinks', []) + refs = data.get("ReferencesAndLinks", []) if isinstance(refs, dict): refs = [x for x in refs.values()] @@ -182,8 +185,9 @@ def _datalad_get(filepath): try: api.get(str(filepath)) except IncompleteResultsError as exc: - if exc.failed[0]['message'] == 'path not associated with any dataset': + if exc.failed[0]["message"] == "path not associated with any dataset": from .conf import TF_GITHUB_SOURCE + api.install(path=TF_LAYOUT.root, source=TF_GITHUB_SOURCE, recursive=True) api.get(str(filepath)) else: @@ -197,23 +201,26 @@ def _s3_get(filepath): import requests path = str(filepath.relative_to(TF_LAYOUT.root)) - url = '%s/%s' % (TF_S3_ROOT, path) + url = "%s/%s" % (TF_S3_ROOT, path) - print('Downloading %s' % url, file=stderr) + print("Downloading %s" % url, file=stderr) # Streaming, so we can iterate over the response. r = requests.get(url, stream=True) # Total size in bytes. - total_size = int(r.headers.get('content-length', 0)) + total_size = int(r.headers.get("content-length", 0)) block_size = 1024 wrote = 0 if not filepath.is_file(): filepath.unlink() - with filepath.open('wb') as f: - for data in tqdm(r.iter_content(block_size), - total=ceil(total_size // block_size), - unit='B', unit_scale=True): + with filepath.open("wb") as f: + for data in tqdm( + r.iter_content(block_size), + total=ceil(total_size // block_size), + unit="B", + unit_scale=True, + ): wrote = wrote + len(data) f.write(data) if total_size != 0 and wrote != total_size: @@ -224,13 +231,15 @@ def _to_bibtex(doi, template, idx): try: from doi2bib.crossref import get_bib_from_doi except ImportError: - print("Cannot generate BibTeX citation, missing doi2bib dependency", - file=sys.stderr) + print( + "Cannot generate BibTeX citation, missing doi2bib dependency", + file=sys.stderr, + ) return doi - if 'doi.org' not in doi: + if "doi.org" not in doi: return doi bib = get_bib_from_doi(doi)[1] # replace identifier with template name - m = re.search(r'([A-Z])\w+', bib) - return bib.replace(m.group(), '%s%s' % (template.lower(), idx)) + m = re.search(r"([A-Z])\w+", bib) + return bib.replace(m.group(), "%s%s" % (template.lower(), idx)) diff --git a/templateflow/conf/__init__.py b/templateflow/conf/__init__.py index 0c3aa4ff..74a13617 100644 --- a/templateflow/conf/__init__.py +++ b/templateflow/conf/__init__.py @@ -2,23 +2,31 @@ from os import getenv from warnings import warn from pathlib import Path -from pkg_resources import resource_filename - -TF_DEFAULT_HOME = Path.home() / '.cache' / 'templateflow' -TF_HOME = Path(getenv('TEMPLATEFLOW_HOME', str(TF_DEFAULT_HOME))) -TF_GITHUB_SOURCE = 'https://github.com/templateflow/templateflow.git' -TF_S3_ROOT = 'https://templateflow.s3.amazonaws.com' -TF_USE_DATALAD = getenv('TEMPLATEFLOW_USE_DATALAD', 'false').lower() in ( - 'true', 'on', '1', 'yes', 'y') + +TF_DEFAULT_HOME = Path.home() / ".cache" / "templateflow" +TF_HOME = Path(getenv("TEMPLATEFLOW_HOME", str(TF_DEFAULT_HOME))) +TF_GITHUB_SOURCE = "https://github.com/templateflow/templateflow.git" +TF_S3_ROOT = "https://templateflow.s3.amazonaws.com" +TF_USE_DATALAD = getenv("TEMPLATEFLOW_USE_DATALAD", "false").lower() in ( + "true", + "on", + "1", + "yes", + "y", +) TF_CACHED = True if not TF_HOME.exists() or not list(TF_HOME.iterdir()): TF_CACHED = False - warn("""\ + warn( + """\ TemplateFlow: repository not found at %s. Populating a new TemplateFlow stub. If the path reported above is not the desired location for TemplateFlow, \ please set the TEMPLATEFLOW_HOME environment variable.\ -""" % TF_HOME, ResourceWarning) +""" + % TF_HOME, + ResourceWarning, + ) if TF_USE_DATALAD: try: from datalad.api import install @@ -29,55 +37,60 @@ install(path=str(TF_HOME), source=TF_GITHUB_SOURCE, recursive=True) if not TF_USE_DATALAD: - from zipfile import ZipFile - TF_HOME.mkdir(exist_ok=True, parents=True) - with ZipFile(resource_filename('templateflow', - 'conf/templateflow-skel.zip'), 'r') as zipref: - zipref.extractall(str(TF_HOME)) + from ._s3 import update as _update_s3 + + _update_s3(TF_HOME, local=True, overwrite=True) -def update_home(force=False): +def update(local=False, overwrite=True): """Update an existing DataLad or S3 home.""" + if TF_USE_DATALAD and _update_datalad(): + return True + + from ._s3 import update as _update_s3 + + return _update_s3(TF_HOME, local=local, overwrite=overwrite) + + +def setup_home(force=False): + """Initialize/update TF's home if necessary.""" if not force and not TF_CACHED: - print("""\ -TemplateFlow was not cached (TEMPLATEFLOW_HOME=%s), \ -a fresh initialization was done.""" % TF_HOME) + print( + f"""\ +TemplateFlow was not cached (TEMPLATEFLOW_HOME={TF_HOME}), \ +a fresh initialization was done.""" + ) return False + return update(local=True, overwrite=False) - if TF_USE_DATALAD: - from datalad.api import update - print("Updating TemplateFlow's HOME using DataLad ...") - try: - update(str(TF_HOME), recursive=True, merge=True) - except Exception as e: - warn("""Error updating TemplateFlow's home directory (using DataLad): -%s""" % str(e)) - return True - # This is an S3 type of installation - from zipfile import ZipFile - with ZipFile(resource_filename('templateflow', - 'conf/templateflow-skel.zip'), 'r') as zipref: - allfiles = zipref.namelist() - current_files = [s.relative_to(TF_HOME) for s in TF_HOME.glob('**/*')] - existing = sorted(set(['%s/' % s.parent for s in current_files])) + \ - [str(s) for s in current_files] - newfiles = sorted(set(allfiles) - set(existing)) - if newfiles: - print("Updating TemplateFlow's HOME using S3. " - "Adding: \n%s" % "\n".join(newfiles)) - zipref.extractall(str(TF_HOME), members=newfiles) - return True - - print("TemplateFlow's HOME directory (S3 type) was up-to-date.") - return False +def _update_datalad(): + from datalad.api import update + + print("Updating TEMPLATEFLOW_HOME using DataLad ...") + try: + update(str(TF_HOME), recursive=True, merge=True) + except Exception as e: + warn(f"Error updating TemplateFlow's home directory (using DataLad): {e}") + return True TF_LAYOUT = None try: from .bids import Layout + TF_LAYOUT = Layout( - TF_HOME, validate=False, config='templateflow', - ignore=['.git', '.datalad', '.gitannex', '.gitattributes', 'scripts']) + TF_HOME, + validate=False, + config="templateflow", + ignore=[ + ".git", + ".datalad", + ".gitannex", + ".gitattributes", + ".github", + "scripts", + ], + ) except ImportError: pass diff --git a/templateflow/conf/_s3.py b/templateflow/conf/_s3.py new file mode 100644 index 00000000..2a76fed0 --- /dev/null +++ b/templateflow/conf/_s3.py @@ -0,0 +1,72 @@ +"""Tooling to handle S3 downloads.""" +from pathlib import Path +from tempfile import mkstemp +from pkg_resources import resource_filename + +TF_SKEL_URL = ( + "https://raw.githubusercontent.com/templateflow/python-client/" + "{release}/templateflow/conf/templateflow-skel.{ext}" +).format +TF_SKEL_PATH = Path(resource_filename("templateflow", "conf/templateflow-skel.zip")) +TF_SKEL_MD5 = Path( + resource_filename("templateflow", "conf/templateflow-skel.md5") +).read_text() + + +def update(dest, local=True, overwrite=True): + """Update an S3-backed TEMPLATEFLOW_HOME repository.""" + skel_file = Path((_get_skeleton_file() if not local else None) or TF_SKEL_PATH) + + retval = _update_skeleton(skel_file, dest, overwrite=overwrite) + if skel_file != TF_SKEL_PATH: + skel_file.unlink() + return retval + + +def _get_skeleton_file(): + import requests + + try: + r = requests.get(TF_SKEL_URL(release="master", ext="md5", allow_redirects=True)) + except requests.exceptions.ConnectionError: + return + + if not r.ok: + return + + if r.content.decode().split()[0] != TF_SKEL_MD5: + r = requests.get(TF_SKEL_URL(release="master", ext="zip", allow_redirects=True)) + if r.ok: + from os import close + + fh, skel_file = mkstemp(suffix=".zip") + Path(skel_file).write_bytes(r.content) + close(fh) + return skel_file + + +def _update_skeleton(skel_file, dest, overwrite=True): + from zipfile import ZipFile + + dest = Path(dest) + dest.mkdir(exist_ok=True, parents=True) + with ZipFile(skel_file, "r") as zipref: + if overwrite: + zipref.extractall(str(dest)) + return True + + allfiles = zipref.namelist() + current_files = [s.relative_to(dest) for s in dest.glob("**/*")] + existing = sorted(set(["%s/" % s.parent for s in current_files])) + [ + str(s) for s in current_files + ] + newfiles = sorted(set(allfiles) - set(existing)) + if newfiles: + print( + "Updating TEMPLATEFLOW_HOME using S3. " + "Adding: \n%s" % "\n".join(newfiles) + ) + zipref.extractall(str(dest), members=newfiles) + return True + print("TEMPLATEFLOW_HOME directory (S3 type) was up-to-date.") + return False diff --git a/templateflow/conf/bids.py b/templateflow/conf/bids.py index 520521e2..a400b588 100644 --- a/templateflow/conf/bids.py +++ b/templateflow/conf/bids.py @@ -2,7 +2,7 @@ from pkg_resources import resource_filename from bids.layout import BIDSLayout, add_config_paths -add_config_paths(templateflow=resource_filename('templateflow', 'conf/config.json')) +add_config_paths(templateflow=resource_filename("templateflow", "conf/config.json")) class Layout(BIDSLayout): @@ -11,5 +11,7 @@ def __repr__(self): s = """\ TemplateFlow Layout - Home: {} - - Templates: {}.""".format(self.root, ', '.join(sorted(self.get_templates()))) + - Templates: {}.""".format( + self.root, ", ".join(sorted(self.get_templates())) + ) return s diff --git a/templateflow/conf/tests/test_conf.py b/templateflow/conf/tests/test_conf.py index e04e4244..72813c13 100644 --- a/templateflow/conf/tests/test_conf.py +++ b/templateflow/conf/tests/test_conf.py @@ -6,12 +6,12 @@ from templateflow import conf as tfc -@pytest.mark.parametrize('use_datalad', ['off', 'on']) +@pytest.mark.parametrize("use_datalad", ["off", "on"]) def test_conf_init(monkeypatch, tmp_path, capsys, use_datalad): """Check the correct functioning of config set-up.""" - home = (tmp_path / '-'.join(('tf', 'dl', use_datalad))).resolve() - monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad) - monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home)) + home = (tmp_path / "-".join(("tf", "dl", use_datalad))).resolve() + monkeypatch.setenv("TEMPLATEFLOW_USE_DATALAD", use_datalad) + monkeypatch.setenv("TEMPLATEFLOW_HOME", str(home)) # First execution, the S3 stub is created (or datalad install) reload(tfc) @@ -23,70 +23,70 @@ def test_conf_init(monkeypatch, tmp_path, capsys, use_datalad): assert str(tfc.TF_HOME) == str(home) -@pytest.mark.parametrize('use_datalad', ['off', 'on']) -def test_update_home(monkeypatch, tmp_path, capsys, use_datalad): +@pytest.mark.parametrize("use_datalad", ["off", "on"]) +def test_setup_home(monkeypatch, tmp_path, capsys, use_datalad): """Check the correct functioning of the installation hook.""" - home = (tmp_path / '-'.join(('tf', 'dl', use_datalad))).resolve() - monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad) - monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home)) + home = (tmp_path / "-".join(("tf", "dl", use_datalad))).resolve() + monkeypatch.setenv("TEMPLATEFLOW_USE_DATALAD", use_datalad) + monkeypatch.setenv("TEMPLATEFLOW_HOME", str(home)) reload(tfc) # First execution, the S3 stub is created (or datalad install) assert tfc.TF_CACHED is False - assert tfc.update_home() is False + assert tfc.setup_home() is False out = capsys.readouterr()[0] - assert out.startswith('TemplateFlow was not cached') - assert ('TEMPLATEFLOW_HOME=%s' % home) in out + assert out.startswith("TemplateFlow was not cached") + assert ("TEMPLATEFLOW_HOME=%s" % home) in out assert home.exists() assert len(list(home.iterdir())) > 0 - updated = tfc.update_home(force=True) # Templateflow is now cached + updated = tfc.setup_home(force=True) # Templateflow is now cached out = capsys.readouterr()[0] - assert not out.startswith('TemplateFlow was not cached') + assert not out.startswith("TemplateFlow was not cached") - if use_datalad == 'on': - assert out.startswith("Updating TemplateFlow's HOME using DataLad") + if use_datalad == "on": + assert out.startswith("Updating TEMPLATEFLOW_HOME using DataLad") assert updated is True - elif use_datalad == 'off': + elif use_datalad == "off": # At this point, S3 should be up-to-date assert updated is False - assert out.startswith("TemplateFlow's HOME directory (S3 type) was up-to-date.") + assert out.startswith("TEMPLATEFLOW_HOME directory (S3 type) was up-to-date.") # Let's force an update - rmtree(str(home / 'tpl-MNI152NLin2009cAsym')) - updated = tfc.update_home(force=True) + rmtree(str(home / "tpl-MNI152NLin2009cAsym")) + updated = tfc.setup_home(force=True) out = capsys.readouterr()[0] assert updated is True - assert out.startswith("Updating TemplateFlow's HOME using S3.") + assert out.startswith("Updating TEMPLATEFLOW_HOME using S3.") reload(tfc) assert tfc.TF_CACHED is True - updated = tfc.update_home() # Templateflow is now cached + updated = tfc.setup_home() # Templateflow is now cached out = capsys.readouterr()[0] - assert not out.startswith('TemplateFlow was not cached') + assert not out.startswith("TemplateFlow was not cached") - if use_datalad == 'on': - assert out.startswith("Updating TemplateFlow's HOME using DataLad") + if use_datalad == "on": + assert out.startswith("Updating TEMPLATEFLOW_HOME using DataLad") assert updated is True - elif use_datalad == 'off': + elif use_datalad == "off": # At this point, S3 should be up-to-date assert updated is False - assert out.startswith("TemplateFlow's HOME directory (S3 type) was up-to-date.") + assert out.startswith("TEMPLATEFLOW_HOME directory (S3 type) was up-to-date.") # Let's force an update - rmtree(str(home / 'tpl-MNI152NLin2009cAsym')) - updated = tfc.update_home() + rmtree(str(home / "tpl-MNI152NLin2009cAsym")) + updated = tfc.setup_home() out = capsys.readouterr()[0] assert updated is True - assert out.startswith("Updating TemplateFlow's HOME using S3.") + assert out.startswith("Updating TEMPLATEFLOW_HOME using S3.") def test_layout(monkeypatch, tmp_path): - monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', 'off') + monkeypatch.setenv("TEMPLATEFLOW_USE_DATALAD", "off") - lines = ('%s' % tfc.TF_LAYOUT).splitlines() + lines = ("%s" % tfc.TF_LAYOUT).splitlines() assert lines[0] == "TemplateFlow Layout" assert lines[1] == " - Home: %s" % tfc.TF_HOME assert lines[2].startswith(" - Templates:") diff --git a/templateflow/conf/tests/test_s3.py b/templateflow/conf/tests/test_s3.py new file mode 100644 index 00000000..10979995 --- /dev/null +++ b/templateflow/conf/tests/test_s3.py @@ -0,0 +1,52 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""Check S3-type repo tooling.""" +# import pytest +from pathlib import Path +import requests +from .. import _s3 as s3 + + +def test_get_skel_file(monkeypatch): + """Exercise the skeleton file generation.""" + local_md5 = s3.TF_SKEL_MD5 + monkeypatch.setattr(s3, "TF_SKEL_MD5", "invent") + new_skel = s3._get_skeleton_file() + assert new_skel is not None + assert Path(new_skel).exists() + assert Path(new_skel).stat().st_size > 0 + + latest_md5 = ( + requests.get(s3.TF_SKEL_URL(release="master", ext="md5", allow_redirects=True)) + .content.decode() + .split()[0] + ) + monkeypatch.setattr(s3, "TF_SKEL_MD5", latest_md5) + assert s3._get_skeleton_file() is None + + monkeypatch.setattr(s3, "TF_SKEL_MD5", local_md5) + monkeypatch.setattr(s3, "TF_SKEL_URL", "http://weird/{release}/{ext}".format) + assert s3._get_skeleton_file() is None + + monkeypatch.setattr( + s3, "TF_SKEL_URL", s3.TF_SKEL_URL(release="{release}", ext="{ext}z").format + ) + assert s3._get_skeleton_file() is None + + +def test_update_s3(tmp_path, monkeypatch): + """Exercise updating the S3 skeleton.""" + newhome = tmp_path / "templateflow" + assert s3.update(newhome) + assert not s3.update(newhome, overwrite=False) + for p in (newhome / "tpl-MNI152NLin6Sym").glob("*.nii.gz"): + p.unlink() + assert s3.update(newhome, overwrite=False) + + # This should cover the remote zip file fetching + monkeypatch.setattr(s3, "TF_SKEL_MD5", "invent") + assert s3.update(newhome, local=False) + assert not s3.update(newhome, local=False, overwrite=False) + for p in (newhome / "tpl-MNI152NLin6Sym").glob("*.nii.gz"): + p.unlink() + assert s3.update(newhome, local=False, overwrite=False) diff --git a/templateflow/tests/test_api.py b/templateflow/tests/test_api.py index d58e8ac1..ba98fc22 100644 --- a/templateflow/tests/test_api.py +++ b/templateflow/tests/test_api.py @@ -5,72 +5,77 @@ # test setup to avoid cluttering pytest parameterize mni2009_urls = [ - 'https://doi.org/10.1016/j.neuroimage.2010.07.033', - 'https://doi.org/10.1016/S1053-8119(09)70884-5', - 'http://nist.mni.mcgill.ca/?p=904', - 'https://doi.org/10.1007/3-540-48714-X_16' + "https://doi.org/10.1016/j.neuroimage.2010.07.033", + "https://doi.org/10.1016/S1053-8119(09)70884-5", + "http://nist.mni.mcgill.ca/?p=904", + "https://doi.org/10.1007/3-540-48714-X_16", ] mni2009_fbib = ( - '@article{mni152nlin2009casym1,\n' - '\tdoi = {10.1016/j.neuroimage.2010.07.033},\n' - '\turl = {https://doi.org/10.1016%2Fj.neuroimage.2010.07.033},\n' - '\tyear = 2011,\n' - '\tmonth = {jan},\n' - '\tpublisher = {Elsevier {BV}},\n' - '\tvolume = {54},\n' - '\tnumber = {1},\n' - '\tpages = {313--327},\n' - '\tauthor = {Vladimir Fonov and Alan C. Evans and Kelly Botteron and C. ' - 'Robert Almli and Robert C. McKinstry and D. Louis Collins},\n' - '\ttitle = {Unbiased average age-appropriate atlases for pediatric studies},\n' - '\tjournal = {{NeuroImage}}\n}' + "@article{mni152nlin2009casym1,\n" + "\tdoi = {10.1016/j.neuroimage.2010.07.033},\n" + "\turl = {https://doi.org/10.1016%2Fj.neuroimage.2010.07.033},\n" + "\tyear = 2011,\n" + "\tmonth = {jan},\n" + "\tpublisher = {Elsevier {BV}},\n" + "\tvolume = {54},\n" + "\tnumber = {1},\n" + "\tpages = {313--327},\n" + "\tauthor = {Vladimir Fonov and Alan C. Evans and Kelly Botteron and C. " + "Robert Almli and Robert C. McKinstry and D. Louis Collins},\n" + "\ttitle = {Unbiased average age-appropriate atlases for pediatric studies},\n" + "\tjournal = {{NeuroImage}}\n}" ) mni2009_lbib = ( - '@incollection{mni152nlin2009casym4,\n' - '\tdoi = {10.1007/3-540-48714-x_16},\n' - '\turl = {https://doi.org/10.1007%2F3-540-48714-x_16},\n' - '\tyear = 1999,\n' - '\tpublisher = {Springer Berlin Heidelberg},\n' - '\tpages = {210--223},\n' - '\tauthor = {D. Louis Collins and Alex P. Zijdenbos and Wim F. C. ' + "@incollection{mni152nlin2009casym4,\n" + "\tdoi = {10.1007/3-540-48714-x_16},\n" + "\turl = {https://doi.org/10.1007%2F3-540-48714-x_16},\n" + "\tyear = 1999,\n" + "\tpublisher = {Springer Berlin Heidelberg},\n" + "\tpages = {210--223},\n" + "\tauthor = {D. Louis Collins and Alex P. Zijdenbos and Wim F. C. " "Baar{\\'{e}} and Alan C. Evans},\n" - '\ttitle = {{ANIMAL}$\\mathplus${INSECT}: Improved Cortical Structure ' - 'Segmentation},\n' - '\tbooktitle = {Lecture Notes in Computer Science}\n}' + "\ttitle = {{ANIMAL}$\\mathplus${INSECT}: Improved Cortical Structure " + "Segmentation},\n" + "\tbooktitle = {Lecture Notes in Computer Science}\n}" ) fslr_urls = [ - 'https://doi.org/10.1093/cercor/bhr291', - 'https://github.com/Washington-University/HCPpipelines/tree/master/global/templates' + "https://doi.org/10.1093/cercor/bhr291", + "https://github.com/Washington-University/HCPpipelines/tree/master/global/templates", ] fslr_fbib = ( - '@article{fslr1,\n' - '\tdoi = {10.1093/cercor/bhr291},\n' - '\turl = {https://doi.org/10.1093%2Fcercor%2Fbhr291},\n' - '\tyear = 2011,\n' - '\tmonth = {nov},\n' - '\tpublisher = {Oxford University Press ({OUP})},\n' - '\tvolume = {22},\n' - '\tnumber = {10},\n' - '\tpages = {2241--2262},\n' - '\tauthor = {D. C. Van Essen and M. F. Glasser and D. L. Dierker and J. ' - 'Harwell and T. Coalson},\n' - '\ttitle = {Parcellations and Hemispheric Asymmetries of Human Cerebral ' - 'Cortex Analyzed on Surface-Based Atlases},\n' - '\tjournal = {Cerebral Cortex}\n}' + "@article{fslr1,\n" + "\tdoi = {10.1093/cercor/bhr291},\n" + "\turl = {https://doi.org/10.1093%2Fcercor%2Fbhr291},\n" + "\tyear = 2011,\n" + "\tmonth = {nov},\n" + "\tpublisher = {Oxford University Press ({OUP})},\n" + "\tvolume = {22},\n" + "\tnumber = {10},\n" + "\tpages = {2241--2262},\n" + "\tauthor = {D. C. Van Essen and M. F. Glasser and D. L. Dierker and J. " + "Harwell and T. Coalson},\n" + "\ttitle = {Parcellations and Hemispheric Asymmetries of Human Cerebral " + "Cortex Analyzed on Surface-Based Atlases},\n" + "\tjournal = {Cerebral Cortex}\n}" ) -fslr_lbib = 'https://github.com/Washington-University/HCPpipelines/tree/master/global/templates' +fslr_lbib = ( + "https://github.com/Washington-University/HCPpipelines/tree/master/global/templates" +) -@pytest.mark.parametrize('template,urls,fbib,lbib', [ - ('MNI152NLin2009cAsym', mni2009_urls, mni2009_fbib, mni2009_lbib), - ('fsLR', fslr_urls, fslr_fbib, fslr_lbib), - ('fsaverage', [], None, None) -]) +@pytest.mark.parametrize( + "template,urls,fbib,lbib", + [ + ("MNI152NLin2009cAsym", mni2009_urls, mni2009_fbib, mni2009_lbib), + ("fsLR", fslr_urls, fslr_fbib, fslr_lbib), + ("fsaverage", [], None, None), + ], +) def test_citations(tmp_path, template, urls, fbib, lbib): assert api.get_citations(template) == urls bibs = api.get_citations(template, bibtex=True) @@ -79,4 +84,4 @@ def test_citations(tmp_path, template, urls, fbib, lbib): assert bibs[-1] == lbib else: # no citations currently - assert template == 'fsaverage' + assert template == "fsaverage" diff --git a/versioneer.py b/versioneer.py index d5471f9d..83b437b3 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,4 +1,3 @@ - # Version: 0.18 """The Versioneer - like a rocketeer, but for versions. @@ -277,6 +276,7 @@ """ from __future__ import print_function + try: import configparser except ImportError: @@ -308,11 +308,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -325,8 +327,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(me)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) except NameError: pass return root @@ -348,6 +352,7 @@ def get(parser, name): if parser.has_option("versioneer", name): return parser.get("versioneer", name) return None + cfg = VersioneerConfig() cfg.VCS = VCS cfg.style = get(parser, "style") or "" @@ -372,17 +377,18 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -390,10 +396,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -418,7 +427,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY[ + "git" +] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -993,7 +1004,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1002,7 +1013,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1010,19 +1021,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1037,8 +1055,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1046,10 +1063,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1072,17 +1098,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -1091,10 +1116,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1105,13 +1132,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1167,16 +1194,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1205,11 +1238,13 @@ def versions_from_file(filename): contents = f.read() except EnvironmentError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1218,8 +1253,7 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1251,8 +1285,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1366,11 +1399,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1390,9 +1425,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1415,8 +1454,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1470,9 +1510,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1524,6 +1568,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in both distutils and setuptools @@ -1542,8 +1587,8 @@ def run(self): # setup.py egg_info -> ? # we override different "build_py" commands for both environments - if 'build_py' in cmds: - _build_py = cmds['build_py'] + if "build_py" in cmds: + _build_py = cmds["build_py"] elif "setuptools" in sys.modules: from setuptools.command.build_py import build_py as _build_py else: @@ -1558,14 +1603,15 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1586,17 +1632,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.distutils_buildexe import py2exe as _py2exe # py3 except ImportError: @@ -1615,18 +1665,22 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments - if 'sdist' in cmds: - _sdist = cmds['sdist'] + if "sdist" in cmds: + _sdist = cmds["sdist"] elif "setuptools" in sys.modules: from setuptools.command.sdist import sdist as _sdist else: @@ -1650,8 +1704,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -1706,11 +1762,13 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except ( + EnvironmentError, + configparser.NoSectionError, + configparser.NoOptionError, + ) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -1719,15 +1777,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -1769,8 +1830,10 @@ def do_setup(): else: print(" 'versioneer.py' already in MANIFEST.in") if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) with open(manifest_in, "a") as f: f.write("include %s\n" % cfg.versionfile_source) else: