ENH: copied materials from datalad@8f17d6ce3e to take advantage of RF…

…ing in docs building etc

docs/source/conf.py

@@ -15,12 +15,25 @@
 import sys
 import os
 import shlex
+from os.path import join as opj, exists
+from os import pardir
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #sys.path.insert(0, os.path.abspath('.'))
 
+# generate missing pieces
+for setup_py_path in (opj(pardir, 'setup.py'),  # travis
+                      opj(pardir, pardir, 'setup.py')):  # RTD
+    if exists(setup_py_path):
+        try:
+            for cmd in 'manpage', 'cfginfo', 'examples':
+                os.system('{} build_{}'.format(setup_py_path, cmd))
+        except:
+            # shut up and do your best
+            pass
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.

readthedocs.yml

@@ -0,0 +1,11 @@
+formats:
+    - epub
+    - pdf
+
+requirements_file: requirements.txt
+
+python:
+   version: 2
+
+python:
+   setup_py_install: true

requirements-devel.txt

@@ -0,0 +1,4 @@
+# Theoretically we don't want -e here but ATM pip would puke if just .[full] is provided
+# Since we use requirements.txt ATM only for development IMHO it is ok but
+# we need to figure out/complaint to pip folks
+-e .[devel]

requirements.txt

@@ -1,11 +1,11 @@
-# Now everything setup says by default for everything
+# If you want to develop, use requirements-devel.txt
+
 # Theoretically we don't want -e here but ATM pip would puke if just .[full] is provided
-# Since we use requirements.txt ATM only for development IMHO it is ok but
-# we need to figure out/complaint to pip folks
--e .[full]
+# TODO -- figure it out and/or complain to pip folks
+# -e .[full]
+
+# this one should work but would copy entire . tree  so should be ran on a clean copy
+.[full]
 
-# -- Optional for Testing
-nose-timer
-# necessary for accessing SecretStorage keyring (system wide Gnome keyring)
-# and nice to have under tox for that
-# dbus-python
+# doesn't install datalad itself
+# file://.#egg=datalad[full]

setup.py

@@ -8,16 +8,31 @@
 
 import platform
 
-from glob import glob
 from os.path import sep as pathsep
+from os.path import join as opj
+from os.path import splitext
+from os.path import dirname
 
+from setuptools import findall
 from setuptools import setup, find_packages
 
 # manpage build imports
-from distutils.command.build_py import build_py
-from setup_support import BuildManPage, BuildRSTExamplesFromScripts
+from setup_support import BuildManPage
+from setup_support import BuildRSTExamplesFromScripts
+from setup_support import BuildConfigInfo
 from setup_support import get_version
 
+
+def findsome(subdir, extensions):
+    """Find files under subdir having specified extensions
+
+    Leading directory (datalad) gets stripped
+    """
+    return [
+        f.split(pathsep, 1)[1] for f in findall(opj('datalad', subdir))
+        if splitext(f)[-1].lstrip('.') in extensions
+    ]
+
 # datalad version to be installed
 version = get_version()
 
@@ -33,17 +48,14 @@
 pbar_requires = ['tqdm']
 
 dist = platform.dist()
-# on oldstable Debian let's ask for lower versions and progressbar instead
-if dist[0] == 'gentoo':
-    pbar_requires = ['progressbar']
+# on oldstable Debian let's ask for lower versions of keyring
 if dist[0] == 'debian' and dist[1].split('.', 1)[0] == '7':
     keyring_requires = ['keyring<8.0']
-    pbar_requires = ['progressbar']
 
 requires = {
     'core': [
         'appdirs',
-        'GitPython>=2.0.3',
+        'GitPython>=2.0.8',
         'iso8601',
         'humanize',
         'mock',  # mock is also used for auto.py, not only for testing
@@ -56,44 +68,93 @@
         'requests>=1.2',
     ] + keyring_requires,
     'downloaders-extra': [
-       'requests_ftp',
+        'requests_ftp',
     ],
     'crawl': [
         'scrapy>=1.1.0rc3',  # versioning is primarily for python3 support
     ],
+    'publish': [
+        'jsmin',             # nice to have, and actually also involved in `install`
+    ],
     'tests': [
         'BeautifulSoup4',  # VERY weak requirement, still used in one of the tests
         'httpretty>=0.8.14',
         'mock',
         'nose>=1.3.4',
-        'testtools',
         'vcrpy',
+    ],
+    'metadata': [
+        'simplejson',
+        'pyld',
+    ],
+    'metadata-extra': [
+        'PyYAML',  # very optional
     ]
 }
+
 requires['full'] = sum(list(requires.values()), [])
 
+# Now add additional ones useful for development
+requires.update({
+    'devel-docs': [
+        # used for converting README.md -> .rst for long_description
+        'pypandoc',
+        # Documentation
+        'sphinx',
+        'sphinx-rtd-theme',
+    ],
+    'devel-utils': [
+        'nose-timer',
+        'line-profiler',
+        # necessary for accessing SecretStorage keyring (system wide Gnome
+        # keyring)  but not installable on travis, IIRC since it needs connectivity
+        # to the dbus whenever installed or smth like that, thus disabled here
+        # but you might need it
+        # 'dbus-python',
+    ],
+    'devel-neuroimaging': [
+        # Specifically needed for tests here (e.g. example scripts testing)
+        'nibabel',
+    ]
+})
+requires['devel'] = sum(list(requires.values()), [])
+
 
+# let's not build manpages and examples automatically (gh-896)
 # configure additional command for custom build steps
-class DataladBuild(build_py):
-    def run(self):
-        self.run_command('build_manpage')
-        self.run_command('build_examples')
-        build_py.run(self)
+#class DataladBuild(build_py):
+#    def run(self):
+#        self.run_command('build_manpage')
+#        self.run_command('build_examples')
+#        build_py.run(self)
 
 cmdclass = {
     'build_manpage': BuildManPage,
     'build_examples': BuildRSTExamplesFromScripts,
-    'build_py': DataladBuild
+    'build_cfginfo': BuildConfigInfo,
+    # 'build_py': DataladBuild
 }
 
+# PyPI doesn't render markdown yet. Workaround for a sane appearance
+# https://github.com/pypa/pypi-legacy/issues/148#issuecomment-227757822
+README = opj(dirname(__file__), 'README.md')
+try:
+    import pypandoc
+    long_description = pypandoc.convert(README, 'rst')
+except ImportError:
+    long_description = open(README).read()
+
 setup(
     name="datalad",
     author="The DataLad Team and Contributors",
     author_email="team@datalad.org",
     version=version,
     description="data distribution geared toward scientific datasets",
+    long_description=long_description,
     packages=datalad_pkgs,
-    install_requires=requires['core'] + requires['downloaders'],
+    install_requires=
+        requires['core'] + requires['downloaders'] +
+        requires['publish'] + requires['metadata'],
     extras_require=requires,
     entry_points={
         'console_scripts': [
@@ -104,11 +165,8 @@ def run(self):
     },
     cmdclass=cmdclass,
     package_data={
-        'datalad': [
-            'resources/git_ssh.sh',
-            'resources/sshserver_cleanup_after_publish.sh',
-            'resources/sshserver_prepare_for_publish.sh',
-        ] +
-        [p.split(pathsep, 1)[1] for p in glob('datalad/downloaders/configs/*.cfg')]
+        'datalad':
+            findsome('resources', {'sh', 'html', 'js', 'css', 'png', 'svg'}) +
+            findsome('downloaders/configs', {'cfg'})
     }
 )

setup_support.py

@@ -15,6 +15,10 @@
 import formatters as fmt
 
 
+def _path_rel2file(p):
+    return opj(dirname(__file__), p)
+
+
 def get_version():
     """Load version of datalad from version.py without entailing any imports
     """
@@ -42,9 +46,9 @@ class BuildManPage(Command):
     ]
 
     def initialize_options(self):
-        self.manpath = None
-        self.rstpath = None
-        self.parser = None
+        self.manpath = opj('build', 'man')
+        self.rstpath = opj('docs', 'source', 'generated', 'man')
+        self.parser = 'datalad.cmdline.main:setup_parser'
 
     def finalize_options(self):
         if self.manpath is None:
@@ -53,6 +57,8 @@ def finalize_options(self):
             raise DistutilsOptionError('\'rstpath\' option is required')
         if self.parser is None:
             raise DistutilsOptionError('\'parser\' option is required')
+        self.manpath = _path_rel2file(self.manpath)
+        self.rstpath = _path_rel2file(self.rstpath)
         mod_name, func_name = self.parser.split(':')
         fromlist = mod_name.split('.')
         try:
@@ -107,15 +113,17 @@ class BuildRSTExamplesFromScripts(Command):
     ]
 
     def initialize_options(self):
-        self.expath = None
-        self.rstpath = None
+        self.expath = opj('docs', 'examples')
+        self.rstpath = opj('docs', 'source', 'generated', 'examples')
 
     def finalize_options(self):
         if self.expath is None:
             raise DistutilsOptionError('\'expath\' option is required')
         if self.rstpath is None:
             raise DistutilsOptionError('\'rstpath\' option is required')
-        self.announce('Converting exanmple scripts')
+        self.expath = _path_rel2file(self.expath)
+        self.rstpath = _path_rel2file(self.rstpath)
+        self.announce('Converting example scripts')
 
     def run(self):
         opath = self.rstpath
@@ -130,3 +138,68 @@ def run(self):
                     open(example),
                     out=out,
                     ref='_example_{0}'.format(exname))
+
+
+class BuildConfigInfo(Command):
+    description = 'Generate RST documentation for all config items.'
+
+    user_options = [
+        ('rstpath=', None, 'output path for RST file'),
+    ]
+
+    def initialize_options(self):
+        self.rstpath = opj('docs', 'source', 'generated', 'cfginfo')
+
+    def finalize_options(self):
+        if self.rstpath is None:
+            raise DistutilsOptionError('\'rstpath\' option is required')
+        self.rstpath = _path_rel2file(self.rstpath)
+        self.announce('Generating configuration documentation')
+
+    def run(self):
+        opath = self.rstpath
+        if not os.path.exists(opath):
+            os.makedirs(opath)
+
+        from datalad.interface.common_cfg import definitions as cfgdefs
+        from datalad.dochelpers import _indent
+
+        categories = {
+            'global': {},
+            'local': {},
+            'dataset': {},
+            'misc': {}
+        }
+        for term, v in cfgdefs.items():
+            categories[v.get('destination', 'misc')][term] = v
+
+        for cat in categories:
+            with open(opj(opath, '{}.rst'.format(cat)), 'w') as rst:
+                rst.write('.. glossary::\n')
+                for term, v in sorted(categories[cat].items(), key=lambda x: x[0]):
+                    rst.write(_indent(term, '\n  '))
+                    qtype, docs = v.get('ui', (None, {}))
+                    desc_tmpl = '\n'
+                    if 'title' in docs:
+                        desc_tmpl += '{title}:\n'
+                    if 'text' in docs:
+                        desc_tmpl += '{text}\n'
+                    if 'default' in v:
+                        default = v['default']
+                        if hasattr(default, 'replace'):
+                            # protect against leaking specific home dirs
+                            v['default'] = default.replace(os.path.expanduser('~'), '~')
+                        desc_tmpl += 'Default: {default}\n'
+                    if 'type' in v:
+                        type_ = v['type']
+                        if hasattr(type_, 'long_description'):
+                            type_ = type_.long_description()
+                        else:
+                            type_ = type_.__name__
+                        desc_tmpl += '\n[{type}]\n'
+                        v['type'] = type_
+                    if desc_tmpl == '\n':
+                        # we need something to avoid joining terms
+                        desc_tmpl += 'undocumented\n'
+                    v.update(docs)
+                    rst.write(_indent(desc_tmpl.format(**v), '    '))