Skip to content
Permalink
Browse files

ENH: copied materials from datalad@8f17d6ce3e to take advantage of RF…

…ing in docs building etc
  • Loading branch information...
yarikoptic committed Oct 17, 2016
1 parent b752ebc commit 24ff4499c90bb3a63e72bd6a855e278504cf5e40
Showing with 197 additions and 38 deletions.
  1. +13 −0 docs/source/conf.py
  2. +11 −0 readthedocs.yml
  3. +4 −0 requirements-devel.txt
  4. +9 −9 requirements.txt
  5. +81 −23 setup.py
  6. +79 −6 setup_support.py
@@ -15,12 +15,25 @@
import sys
import os
import shlex
from os.path import join as opj, exists
from os import pardir

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# generate missing pieces
for setup_py_path in (opj(pardir, 'setup.py'), # travis
opj(pardir, pardir, 'setup.py')): # RTD
if exists(setup_py_path):
try:
for cmd in 'manpage', 'cfginfo', 'examples':
os.system('{} build_{}'.format(setup_py_path, cmd))
except:
# shut up and do your best
pass

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
@@ -0,0 +1,11 @@
formats:
- epub
- pdf

requirements_file: requirements.txt

python:
version: 2

python:
setup_py_install: true
@@ -0,0 +1,4 @@
# Theoretically we don't want -e here but ATM pip would puke if just .[full] is provided
# Since we use requirements.txt ATM only for development IMHO it is ok but
# we need to figure out/complaint to pip folks
-e .[devel]
@@ -1,11 +1,11 @@
# Now everything setup says by default for everything
# If you want to develop, use requirements-devel.txt

# Theoretically we don't want -e here but ATM pip would puke if just .[full] is provided
# Since we use requirements.txt ATM only for development IMHO it is ok but
# we need to figure out/complaint to pip folks
-e .[full]
# TODO -- figure it out and/or complain to pip folks
# -e .[full]

# this one should work but would copy entire . tree so should be ran on a clean copy
.[full]

# -- Optional for Testing
nose-timer
# necessary for accessing SecretStorage keyring (system wide Gnome keyring)
# and nice to have under tox for that
# dbus-python
# doesn't install datalad itself
# file://.#egg=datalad[full]
104 setup.py
@@ -8,16 +8,31 @@

import platform

from glob import glob
from os.path import sep as pathsep
from os.path import join as opj
from os.path import splitext
from os.path import dirname

from setuptools import findall
from setuptools import setup, find_packages

# manpage build imports
from distutils.command.build_py import build_py
from setup_support import BuildManPage, BuildRSTExamplesFromScripts
from setup_support import BuildManPage
from setup_support import BuildRSTExamplesFromScripts
from setup_support import BuildConfigInfo
from setup_support import get_version


def findsome(subdir, extensions):
"""Find files under subdir having specified extensions
Leading directory (datalad) gets stripped
"""
return [
f.split(pathsep, 1)[1] for f in findall(opj('datalad', subdir))
if splitext(f)[-1].lstrip('.') in extensions
]

# datalad version to be installed
version = get_version()

@@ -33,17 +48,14 @@
pbar_requires = ['tqdm']

dist = platform.dist()
# on oldstable Debian let's ask for lower versions and progressbar instead
if dist[0] == 'gentoo':
pbar_requires = ['progressbar']
# on oldstable Debian let's ask for lower versions of keyring
if dist[0] == 'debian' and dist[1].split('.', 1)[0] == '7':
keyring_requires = ['keyring<8.0']
pbar_requires = ['progressbar']

requires = {
'core': [
'appdirs',
'GitPython>=2.0.3',
'GitPython>=2.0.8',
'iso8601',
'humanize',
'mock', # mock is also used for auto.py, not only for testing
@@ -56,44 +68,93 @@
'requests>=1.2',
] + keyring_requires,
'downloaders-extra': [
'requests_ftp',
'requests_ftp',
],
'crawl': [
'scrapy>=1.1.0rc3', # versioning is primarily for python3 support
],
'publish': [
'jsmin', # nice to have, and actually also involved in `install`
],
'tests': [
'BeautifulSoup4', # VERY weak requirement, still used in one of the tests
'httpretty>=0.8.14',
'mock',
'nose>=1.3.4',
'testtools',
'vcrpy',
],
'metadata': [
'simplejson',
'pyld',
],
'metadata-extra': [
'PyYAML', # very optional
]
}

requires['full'] = sum(list(requires.values()), [])

# Now add additional ones useful for development
requires.update({
'devel-docs': [
# used for converting README.md -> .rst for long_description
'pypandoc',
# Documentation
'sphinx',
'sphinx-rtd-theme',
],
'devel-utils': [
'nose-timer',
'line-profiler',
# necessary for accessing SecretStorage keyring (system wide Gnome
# keyring) but not installable on travis, IIRC since it needs connectivity
# to the dbus whenever installed or smth like that, thus disabled here
# but you might need it
# 'dbus-python',
],
'devel-neuroimaging': [
# Specifically needed for tests here (e.g. example scripts testing)
'nibabel',
]
})
requires['devel'] = sum(list(requires.values()), [])


# let's not build manpages and examples automatically (gh-896)
# configure additional command for custom build steps
class DataladBuild(build_py):
def run(self):
self.run_command('build_manpage')
self.run_command('build_examples')
build_py.run(self)
#class DataladBuild(build_py):
# def run(self):
# self.run_command('build_manpage')
# self.run_command('build_examples')
# build_py.run(self)

cmdclass = {
'build_manpage': BuildManPage,
'build_examples': BuildRSTExamplesFromScripts,
'build_py': DataladBuild
'build_cfginfo': BuildConfigInfo,
# 'build_py': DataladBuild
}

# PyPI doesn't render markdown yet. Workaround for a sane appearance
# https://github.com/pypa/pypi-legacy/issues/148#issuecomment-227757822
README = opj(dirname(__file__), 'README.md')
try:
import pypandoc
long_description = pypandoc.convert(README, 'rst')
except ImportError:
long_description = open(README).read()

setup(
name="datalad",
author="The DataLad Team and Contributors",
author_email="team@datalad.org",
version=version,
description="data distribution geared toward scientific datasets",
long_description=long_description,
packages=datalad_pkgs,
install_requires=requires['core'] + requires['downloaders'],
install_requires=
requires['core'] + requires['downloaders'] +
requires['publish'] + requires['metadata'],
extras_require=requires,
entry_points={
'console_scripts': [
@@ -104,11 +165,8 @@ def run(self):
},
cmdclass=cmdclass,
package_data={
'datalad': [
'resources/git_ssh.sh',
'resources/sshserver_cleanup_after_publish.sh',
'resources/sshserver_prepare_for_publish.sh',
] +
[p.split(pathsep, 1)[1] for p in glob('datalad/downloaders/configs/*.cfg')]
'datalad':
findsome('resources', {'sh', 'html', 'js', 'css', 'png', 'svg'}) +
findsome('downloaders/configs', {'cfg'})
}
)
@@ -15,6 +15,10 @@
import formatters as fmt


def _path_rel2file(p):
return opj(dirname(__file__), p)


def get_version():
"""Load version of datalad from version.py without entailing any imports
"""
@@ -42,9 +46,9 @@ class BuildManPage(Command):
]

def initialize_options(self):
self.manpath = None
self.rstpath = None
self.parser = None
self.manpath = opj('build', 'man')
self.rstpath = opj('docs', 'source', 'generated', 'man')
self.parser = 'datalad.cmdline.main:setup_parser'

def finalize_options(self):
if self.manpath is None:
@@ -53,6 +57,8 @@ def finalize_options(self):
raise DistutilsOptionError('\'rstpath\' option is required')
if self.parser is None:
raise DistutilsOptionError('\'parser\' option is required')
self.manpath = _path_rel2file(self.manpath)
self.rstpath = _path_rel2file(self.rstpath)
mod_name, func_name = self.parser.split(':')
fromlist = mod_name.split('.')
try:
@@ -107,15 +113,17 @@ class BuildRSTExamplesFromScripts(Command):
]

def initialize_options(self):
self.expath = None
self.rstpath = None
self.expath = opj('docs', 'examples')
self.rstpath = opj('docs', 'source', 'generated', 'examples')

def finalize_options(self):
if self.expath is None:
raise DistutilsOptionError('\'expath\' option is required')
if self.rstpath is None:
raise DistutilsOptionError('\'rstpath\' option is required')
self.announce('Converting exanmple scripts')
self.expath = _path_rel2file(self.expath)
self.rstpath = _path_rel2file(self.rstpath)
self.announce('Converting example scripts')

def run(self):
opath = self.rstpath
@@ -130,3 +138,68 @@ def run(self):
open(example),
out=out,
ref='_example_{0}'.format(exname))


class BuildConfigInfo(Command):
description = 'Generate RST documentation for all config items.'

user_options = [
('rstpath=', None, 'output path for RST file'),
]

def initialize_options(self):
self.rstpath = opj('docs', 'source', 'generated', 'cfginfo')

def finalize_options(self):
if self.rstpath is None:
raise DistutilsOptionError('\'rstpath\' option is required')
self.rstpath = _path_rel2file(self.rstpath)
self.announce('Generating configuration documentation')

def run(self):
opath = self.rstpath
if not os.path.exists(opath):
os.makedirs(opath)

from datalad.interface.common_cfg import definitions as cfgdefs
from datalad.dochelpers import _indent

categories = {
'global': {},
'local': {},
'dataset': {},
'misc': {}
}
for term, v in cfgdefs.items():
categories[v.get('destination', 'misc')][term] = v

for cat in categories:
with open(opj(opath, '{}.rst'.format(cat)), 'w') as rst:
rst.write('.. glossary::\n')
for term, v in sorted(categories[cat].items(), key=lambda x: x[0]):
rst.write(_indent(term, '\n '))
qtype, docs = v.get('ui', (None, {}))
desc_tmpl = '\n'
if 'title' in docs:
desc_tmpl += '{title}:\n'
if 'text' in docs:
desc_tmpl += '{text}\n'
if 'default' in v:
default = v['default']
if hasattr(default, 'replace'):
# protect against leaking specific home dirs
v['default'] = default.replace(os.path.expanduser('~'), '~')
desc_tmpl += 'Default: {default}\n'
if 'type' in v:
type_ = v['type']
if hasattr(type_, 'long_description'):
type_ = type_.long_description()
else:
type_ = type_.__name__
desc_tmpl += '\n[{type}]\n'
v['type'] = type_
if desc_tmpl == '\n':
# we need something to avoid joining terms
desc_tmpl += 'undocumented\n'
v.update(docs)
rst.write(_indent(desc_tmpl.format(**v), ' '))

0 comments on commit 24ff449

Please sign in to comment.
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.