Skip to content

Commit

Permalink
dist.sphinxext: new sphinx extension
Browse files Browse the repository at this point in the history
Small sphinx extension to generate docs from argparse scripts.
Simplifies all `conf.py` across all pkgcore stack.

Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>
  • Loading branch information
arthurzam committed Nov 11, 2022
1 parent f3486bd commit 1bd48a4
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 112 deletions.
2 changes: 1 addition & 1 deletion .coveragerc
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[run]
source = snakeoil
branch = True
omit = src/*, tests/*
omit = src/*, tests/*, src/snakeoil/dist/*

[paths]
source = **/site-packages/snakeoil
Expand Down
105 changes: 2 additions & 103 deletions src/snakeoil/dist/distutils_extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import sys
import textwrap
from contextlib import ExitStack, contextmanager, redirect_stderr, redirect_stdout
from datetime import datetime
from multiprocessing import cpu_count

from setuptools import find_packages
Expand Down Expand Up @@ -120,38 +119,8 @@ def module_version(moduledir=MODULEDIR):
Based on the assumption that a module defines __version__.
"""
version = None
try:
with open(os.path.join(moduledir, '__init__.py'), encoding='utf-8') as f:
version = re.search(
r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
f.read(), re.MULTILINE).group(1)
except IOError as e:
if e.errno == errno.ENOENT:
pass
else:
raise

if version is None:
raise RuntimeError(f'Cannot find version for module: {MODULE_NAME}')

# use versioning scheme similar to setuptools_scm for untagged versions
git_version = get_git_version(REPODIR)
if git_version:
tag = git_version['tag']
if tag is None:
commits = git_version['commits']
rev = git_version['rev'][:7]
date = datetime.strptime(git_version['date'], '%a, %d %b %Y %H:%M:%S %z')
date = datetime.strftime(date, '%Y%m%d')
if commits is not None:
version += f'.dev{commits}'
version += f'+g{rev}.d{date}'
elif tag != version:
raise DistutilsError(
f'unmatched git tag {tag!r} and {MODULE_NAME} version {version!r}')

return version
from .utilities import module_version
return module_version(REPODIR, moduledir)


def generate_verinfo(target_dir):
Expand Down Expand Up @@ -266,76 +235,6 @@ def data_mapping(host_prefix, path, skip=None):
if os.path.join(root, x) not in skip])


def pkg_config(*packages, **kw):
"""Translate pkg-config data to compatible Extension parameters.
Example usage:
>>> from distutils.extension import Extension
>>> from pkgdist import pkg_config
>>>
>>> ext_kwargs = dict(
... include_dirs=['include'],
... extra_compile_args=['-std=c++11'],
... )
>>> extensions = [
... Extension('foo', ['foo.c']),
... Extension('bar', ['bar.c'], **pkg_config('lcms2')),
... Extension('ext', ['ext.cpp'], **pkg_config(('nss', 'libusb-1.0'), **ext_kwargs)),
... ]
"""
flag_map = {
'-I': 'include_dirs',
'-L': 'library_dirs',
'-l': 'libraries',
}

try:
tokens = subprocess.check_output(
['pkg-config', '--libs', '--cflags'] + list(packages)).split()
except OSError as e:
sys.stderr.write(f'running pkg-config failed: {e.strerror}\n')
sys.exit(1)

for token in tokens:
token = token.decode()
if token[:2] in flag_map:
kw.setdefault(flag_map.get(token[:2]), []).append(token[2:])
else:
kw.setdefault('extra_compile_args', []).append(token)
return kw


def cython_pyx(path=MODULEDIR):
"""Return all available cython extensions under a given path."""
for root, _dirs, files in os.walk(path):
for f in files:
if f.endswith('.pyx'):
yield str(os.path.join(root, f))


def cython_exts(path=MODULEDIR, build_opts=None):
"""Prepare all cython extensions under a given path to be built."""
if build_opts is None:
build_opts = {'depends': [], 'include_dirs': []}
exts = []

for ext in cython_pyx(path):
cythonized = os.path.splitext(ext)[0] + '.c'
if os.path.exists(cythonized):
ext_path = cythonized
else:
ext_path = ext

# strip package dir
module = ext_path.rpartition(PACKAGEDIR)[-1].lstrip(os.path.sep)
# strip file extension and translate to module namespace
module = os.path.splitext(module)[0].replace(os.path.sep, '.')
exts.append(Extension(module, [ext_path], **build_opts))

return exts


class sdist(dst_sdist.sdist):
"""sdist command wrapper to bundle generated files for release."""

Expand Down
14 changes: 6 additions & 8 deletions src/snakeoil/dist/generate_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ def _generate_custom(project, docdir, gendir):
custom_dir = os.path.join(docdir, 'generate')
print(f"Generating custom docs for {project} in {gendir!r}")

for root, dirs, files in os.walk(custom_dir):
for root, _dirs, files in os.walk(custom_dir):
subdir = root.split(custom_dir, 1)[1].strip('/')
if subdir:
try:
os.mkdir(os.path.join(gendir, subdir))
except OSError as e:
if e.errno != errno.EEXIST:
except OSError as exc:
if exc.errno != errno.EEXIST:
raise

for script in sorted(x for x in files if not x.startswith(('.', '_'))):
Expand All @@ -44,8 +44,7 @@ def _generate_custom(project, docdir, gendir):
module.main(fake_file, docdir=docdir, gendir=gendir)

fake_file.seek(0)
data = fake_file.read()
if data:
if data := fake_file.read():
rst = os.path.join(gendir, subdir, os.path.splitext(script)[0] + '.rst')
print(f"generating {rst}")
with open(rst, 'w') as f:
Expand All @@ -66,7 +65,7 @@ def generate_man(repo_dir, package_dir, module):

# Replace '-' with '_' due to python namespace contraints.
generated_man_pages = [
('%s.scripts.' % (module) + s.replace('-', '_'), s) for s in scripts
(f"{module}.scripts.{s.replace('-', '_')}", s) for s in scripts
]

# generate specified man pages for scripts
Expand All @@ -88,5 +87,4 @@ def generate_html(repo_dir, package_dir, module):
os.path.join(package_dir, module),
os.path.join(package_dir, module, 'test'),
os.path.join(package_dir, module, 'scripts')]):
raise RuntimeError(
'API doc generation failed for %s' % (module,))
raise RuntimeError(f'API doc generation failed for {module}')
80 changes: 80 additions & 0 deletions src/snakeoil/dist/sphinxext.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
"""small sphinx extension to generate docs from argparse scripts"""

import sys
from importlib import import_module
from pathlib import Path

from sphinx.application import Sphinx
from sphinx.ext.apidoc import main as sphinx_apidoc

from .generate_docs import _generate_custom
from .generate_man_rsts import ManConverter
from .utilities import module_version

if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib


def prepare_scripts_man(repo_dir: Path, man_pages: list[tuple]):
# Workaround for sphinx doing include directive path mangling in
# order to interpret absolute paths "correctly", but at the same
# time causing relative paths to fail. This just bypasses the
# sphinx mangling and lets docutils handle include directives
# directly which works as expected.
from docutils.parsers.rst.directives.misc import Include as BaseInclude
from sphinx.directives.other import Include
Include.run = BaseInclude.run

with open(repo_dir / 'pyproject.toml', 'rb') as file:
pyproj = tomllib.load(file)

authors_list = [
f'{author["name"]} <{author["email"]}>' for author in pyproj['project']['authors']
]

for i, man_page in enumerate(man_pages):
if man_page[3] is None:
m = list(man_page)
m[3] = authors_list
man_pages[i] = tuple(m)

man_gen_dir = str(repo_dir / 'doc' / 'generated')

for name, entry in pyproj['project']['scripts'].items():
module: str = entry.split(':')[0]
man_pages.append((f'man/{name}', name, import_module(module).__doc__.strip().split('\n', 1)[0], authors_list, 1))
ManConverter.regen_if_needed(man_gen_dir, module.replace('__init__', name), out_name=name)


def generate_html(repo_dir: Path, module: str):
"""Generate API rst docs for a project.
This uses sphinx-apidoc to auto-generate all the required rst files.
"""
apidir = repo_dir / 'doc' / 'api'
package_dir = repo_dir / 'src' / module
sphinx_apidoc(['-Tef', '-o', str(apidir),
str(package_dir), str(package_dir / 'test'),
str(package_dir / 'scripts')])


def doc_backend(app: Sphinx):
repo_dir = Path(app.config.repodir)
if not app.config.version:
app.config.version = module_version(repo_dir, repo_dir / 'src' / app.config.project)

prepare_scripts_man(repo_dir, app.config.man_pages)

if app.builder.name in ('man', 'html'):
docdir = repo_dir / 'doc'
_generate_custom(app.config.project, str(docdir), str(docdir / 'generated'))

if app.builder.name == 'html':
generate_html(repo_dir, app.config.project)


def setup(app: Sphinx):
app.connect('builder-inited', doc_backend)
app.add_config_value(name="repodir", default=Path.cwd(), rebuild=True)
44 changes: 44 additions & 0 deletions src/snakeoil/dist/utilities.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import errno
import os
import re
from datetime import datetime

from ..version import get_git_version

def module_version(repodir, moduledir):
"""Determine a module's version.
Based on the assumption that a module defines __version__.
"""
version = None
try:
with open(os.path.join(moduledir, '__init__.py'), encoding='utf-8') as f:
version = re.search(
r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
f.read(), re.MULTILINE).group(1)
except IOError as exc:
if exc.errno == errno.ENOENT:
pass
else:
raise

if version is None:
raise RuntimeError(f'Cannot find version for module in: {moduledir}')

# use versioning scheme similar to setuptools_scm for untagged versions
git_version = get_git_version(str(repodir))
if git_version:
tag = git_version['tag']
if tag is None:
commits = git_version['commits']
rev = git_version['rev'][:7]
date = datetime.strptime(git_version['date'], '%a, %d %b %Y %H:%M:%S %z')
date = datetime.strftime(date, '%Y%m%d')
if commits is not None:
version += f'.dev{commits}'
version += f'+g{rev}.d{date}'
elif tag != version:
raise RuntimeError(
f'unmatched git tag {tag!r} and module version {version!r}')

return version

0 comments on commit 1bd48a4

Please sign in to comment.