Skip to content

Commit

Permalink
Isolated building of DocumentReleases in a separate update_docs method
Browse files Browse the repository at this point in the history
  • Loading branch information
@claudep
claudep committed May 1, 2018
1 parent d3ac98f commit 1e19b77
Showing 1 changed file with 124 additions and 121 deletions.
245 changes: 124 additions & 121 deletions docs/management/commands/update_docs.py
View file
Expand Up @@ -28,143 +28,146 @@ def add_arguments(self, parser):
)

def handle(self, **kwargs):
self.verbosity = verbosity = kwargs['verbosity']
update_index = kwargs['update_index']
self.verbosity = kwargs['verbosity']
self.update_index = kwargs['update_index']

default_builders = ['json', 'djangohtml']
self.default_builders = ['json', 'djangohtml']
default_docs_version = DocumentRelease.objects.get(is_default=True).release.version

# Keep track of which Git sources have been updated.
release_docs_changed = {} # e.g. {'1.8': True} if the 1.8 docs updated.
self.release_docs_changed = {} # e.g. {'1.8': True} if the 1.8 docs updated.
# Only update the index if some docs rebuild.
update_index_required = False
self.update_index_required = False

# Somehow, bizarely, there's a bug in Sphinx such that if I try to
# build 1.0 before other versions, things fail in weird ways. However,
# building newer versions first works. I suspect Sphinx is hanging onto
# some global state. Anyway, we can work around it by making sure that
# "dev" builds before "1.0". This is ugly, but oh well.
for release in DocumentRelease.objects.order_by('-release'):
if verbosity >= 1:
self.stdout.write("Updating %s..." % release)

# checkout_dir is shared for all languages.
checkout_dir = settings.DOCS_BUILD_ROOT.joinpath('sources', release.version)
parent_build_dir = settings.DOCS_BUILD_ROOT.joinpath(release.lang, release.version)
if not checkout_dir.exists():
checkout_dir.mkdir(parents=True)
if not parent_build_dir.exists():
parent_build_dir.mkdir(parents=True)

#
# Update the release from SCM.
#
# Make a git checkout/update into the destination directory.
if (not self.update_git(release.scm_url, checkout_dir, changed_dir='docs/') and
not release_docs_changed.get(release.version)):
# No docs changes so don't rebuild.
# Skip translated non-stable versions to avoid a crash:
# https://github.com/django/djangoproject.com/issues/627
if release.lang != 'en' and not release.release.version == default_docs_version:
continue

update_index_required = update_index
release_docs_changed[release.version] = True

source_dir = checkout_dir.joinpath('docs')

if release.lang != 'en':
# Skip translated non-stable versions to avoid a crash:
# https://github.com/django/djangoproject.com/issues/627
if not release.release.version == default_docs_version:
continue
scm_url = release.scm_url.replace('django.git', 'django-docs-translations.git')
trans_dir = checkout_dir.joinpath('django-docs-translation')
if not trans_dir.exists():
trans_dir.mkdir()
self.update_git(scm_url, trans_dir)
if not source_dir.joinpath('locale').exists():
source_dir.joinpath('locale').symlink_to(trans_dir.joinpath('translations'))
extra_kwargs = {'stdout': subprocess.DEVNULL} if verbosity == 0 else {}
subprocess.call('cd %s && make translations' % trans_dir, shell=True, **extra_kwargs)

if release.is_default:
# Build the pot files (later retrieved by Transifex)
builders = default_builders[:] + ['gettext']
else:
builders = default_builders

#
# Use Sphinx to build the release docs into JSON and HTML documents.
#
for builder in builders:
# Wipe and re-create the build directory. See #18930.
build_dir = parent_build_dir.joinpath('_build', builder)
if build_dir.exists():
shutil.rmtree(str(build_dir))
build_dir.mkdir(parents=True)

if verbosity >= 2:
self.stdout.write(" building %s (%s -> %s)" % (builder, source_dir, build_dir))
subprocess.check_call([
'sphinx-build',
'-b', builder,
'-D', 'language=%s' % to_locale(release.lang),
'-Q' if verbosity == 0 else '-q',
str(source_dir), # Source file directory
str(build_dir), # Destination directory
])

#
# Create a zip file of the HTML build for offline reading.
# This gets moved into MEDIA_ROOT for downloading.
#
html_build_dir = parent_build_dir.joinpath('_build', 'djangohtml')
zipfile_name = 'django-docs-%s-%s.zip' % (release.version, release.lang)
zipfile_path = Path(settings.MEDIA_ROOT).joinpath('docs', zipfile_name)
if not zipfile_path.parent.exists():
zipfile_path.parent.mkdir(parents=True)
if verbosity >= 2:
self.stdout.write(" build zip (into %s)" % zipfile_path)

def zipfile_inclusion_filter(file_path):
return '.doctrees' not in file_path.parts

with closing(zipfile.ZipFile(str(zipfile_path), 'w', compression=zipfile.ZIP_DEFLATED)) as zf:
for root, dirs, files in os.walk(str(html_build_dir)):
for f in files:
file_path = Path(os.path.join(root, f))
if zipfile_inclusion_filter(file_path):
rel_path = str(file_path.relative_to(html_build_dir))
zf.write(str(file_path), rel_path)

#
# Copy the build results to the directory used for serving
# the documentation in the least disruptive way possible.
#
build_dir = parent_build_dir.joinpath('_build')
built_dir = parent_build_dir.joinpath('_built')
self.build_doc_release(release)

if self.update_index_required:
call_command('update_index', **{'verbosity': self.verbosity})

def build_doc_release(self, release):
if self.verbosity >= 1:
self.stdout.write("Updating %s..." % release)

# checkout_dir is shared for all languages.
checkout_dir = settings.DOCS_BUILD_ROOT.joinpath('sources', release.version)
parent_build_dir = settings.DOCS_BUILD_ROOT.joinpath(release.lang, release.version)
if not checkout_dir.exists():
checkout_dir.mkdir(parents=True)
if not parent_build_dir.exists():
parent_build_dir.mkdir(parents=True)

#
# Update the release from SCM.
#
# Make a git checkout/update into the destination directory.
if (not self.update_git(release.scm_url, checkout_dir, changed_dir='docs/') and
not self.release_docs_changed.get(release.version)):
# No docs changes so don't rebuild.
return

self.update_index_required = self.update_index
self.release_docs_changed[release.version] = True

source_dir = checkout_dir.joinpath('docs')

if release.lang != 'en':
scm_url = release.scm_url.replace('django.git', 'django-docs-translations.git')
trans_dir = checkout_dir.joinpath('django-docs-translation')
if not trans_dir.exists():
trans_dir.mkdir()
self.update_git(scm_url, trans_dir)
if not source_dir.joinpath('locale').exists():
source_dir.joinpath('locale').symlink_to(trans_dir.joinpath('translations'))
extra_kwargs = {'stdout': subprocess.DEVNULL} if self.verbosity == 0 else {}
subprocess.call('cd %s && make translations' % trans_dir, shell=True, **extra_kwargs)

if release.is_default:
# Build the pot files (later retrieved by Transifex)
builders = self.default_builders[:] + ['gettext']
else:
builders = self.default_builders

#
# Use Sphinx to build the release docs into JSON and HTML documents.
#
for builder in builders:
# Wipe and re-create the build directory. See #18930.
build_dir = parent_build_dir.joinpath('_build', builder)
if build_dir.exists():
shutil.rmtree(str(build_dir))
build_dir.mkdir(parents=True)

if self.verbosity >= 2:
self.stdout.write(" building %s (%s -> %s)" % (builder, source_dir, build_dir))
subprocess.check_call([
'rsync',
'--archive',
'--delete',
'--link-dest={}'.format(build_dir),
'{}/'.format(build_dir),
str(built_dir)
'sphinx-build',
'-b', builder,
'-D', 'language=%s' % to_locale(release.lang),
'-Q' if self.verbosity == 0 else '-q',
str(source_dir), # Source file directory
str(build_dir), # Destination directory
])

#
# Rebuild the imported document list and search index.
#
if not update_index:
continue

if verbosity >= 2:
self.stdout.write(" reindexing...")

json_built_dir = parent_build_dir.joinpath('_built', 'json')
documents = gen_decoded_documents(json_built_dir)
release.sync_to_db(documents)

if update_index_required:
call_command('update_index', **{'verbosity': kwargs['verbosity']})
#
# Create a zip file of the HTML build for offline reading.
# This gets moved into MEDIA_ROOT for downloading.
#
html_build_dir = parent_build_dir.joinpath('_build', 'djangohtml')
zipfile_name = 'django-docs-%s-%s.zip' % (release.version, release.lang)
zipfile_path = Path(settings.MEDIA_ROOT).joinpath('docs', zipfile_name)
if not zipfile_path.parent.exists():
zipfile_path.parent.mkdir(parents=True)
if self.verbosity >= 2:
self.stdout.write(" build zip (into %s)" % zipfile_path)

def zipfile_inclusion_filter(file_path):
return '.doctrees' not in file_path.parts

with closing(zipfile.ZipFile(str(zipfile_path), 'w', compression=zipfile.ZIP_DEFLATED)) as zf:
for root, dirs, files in os.walk(str(html_build_dir)):
for f in files:
file_path = Path(os.path.join(root, f))
if zipfile_inclusion_filter(file_path):
rel_path = str(file_path.relative_to(html_build_dir))
zf.write(str(file_path), rel_path)

#
# Copy the build results to the directory used for serving
# the documentation in the least disruptive way possible.
#
build_dir = parent_build_dir.joinpath('_build')
built_dir = parent_build_dir.joinpath('_built')
subprocess.check_call([
'rsync',
'--archive',
'--delete',
'--link-dest={}'.format(build_dir),
'{}/'.format(build_dir),
str(built_dir)
])

#
# Rebuild the imported document list and search index.
#
if not self.update_index:
return

if self.verbosity >= 2:
self.stdout.write(" reindexing...")

json_built_dir = parent_build_dir.joinpath('_built', 'json')
documents = gen_decoded_documents(json_built_dir)
release.sync_to_db(documents)

def update_git(self, url, destdir, changed_dir='.'):
"""
Expand Down

0 comments on commit 1e19b77

Please sign in to comment.