Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: 7e8c265cc2
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

file 181 lines (157 sloc) 7.487 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
"""
Update and build the documentation into files for display with the djangodocs
app.
"""
from __future__ import absolute_import

import os
import json
import haystack
import optparse
import subprocess
import zipfile
import sphinx.cmdline
from contextlib import closing
from django.conf import settings
from django.core.management.base import NoArgsCommand
from django.utils.html import strip_tags
from django.utils.text import unescape_entities
from unipath import FSPath as Path
from ...models import DocumentRelease, Document

class Command(NoArgsCommand):
    option_list = NoArgsCommand.option_list + (
        optparse.make_option(
            '--skip-indexing',
            action='store_false',
            dest='reindex',
            default=True,
            help='Skip reindexing (for testing, mostly).'
        ),
    )

    def handle_noargs(self, **kwargs):
        try:
            verbosity = int(kwargs['verbosity'])
        except (KeyError, TypeError, ValueError):
            verbosity = 1

        # Somehow, bizarely, there's a bug in Sphinx such that if I try to
        # build 1.0 before other versions, things fail in weird ways. However,
        # building newer versions first works. I suspect Sphinx is hanging onto
        # some global state. Anyway, we can work around it by making sure that
        # "dev" builds before "1.0". This is ugly, but oh well.
        for release in DocumentRelease.objects.order_by('-version'):
            if verbosity >= 1:
                print "Updating %s..." % release

            destdir = Path(settings.DOCS_BUILD_ROOT).child(release.lang, release.version)
            if not destdir.exists():
                destdir.mkdir(parents=True)

            #
            # Update the release from SCM.
            #

            # Make an SCM checkout/update into the destination directory.
            # Do this dynamically in case we add other SCM later.
            getattr(self, 'update_%s' % release.scm)(release.scm_url, destdir)

            #
            # Use Sphinx to build the release docs into JSON and HTML documents.
            #
            if release.docs_subdir:
                source_dir = destdir.child(*release.docs_subdir.split('/'))
            else:
                source_dir = destdir

            for builder in ('json', 'html'):
                # Make the directory for the built files - sphinx-build doesn't
                # do it for us, apparently.
                build_dir = destdir.child('_build', builder)
                if not build_dir.exists():
                    build_dir.mkdir(parents=True)

                # "Shell out" (not exactly, but basically) to sphinx-build.
                if verbosity >= 2:
                    print " building %s (%s -> %s)" % (builder, source_dir, build_dir)
                sphinx.cmdline.main(['sphinx-build',
                    '-b', builder,
                    '-q', # Be vewy qwiet
                    source_dir, # Source file directory
                    build_dir, # Destination directory
                ])

            #
            # Create a zip file of the HTML build for offline reading.
            # This gets moved into MEDIA_ROOT for downloading.
            #
            html_build_dir = destdir.child('_build', 'html')
            zipfile_name = 'django-docs-%s-%s.zip' % (release.version, release.lang)
            zipfile_path = Path(settings.MEDIA_ROOT).child('docs', zipfile_name)
            if not zipfile_path.parent.exists():
                zipfile_path.parent.mkdir(parents=True)
            if verbosity >= 2:
                print " build zip (into %s)" % zipfile_path

            def zipfile_inclusion_filter(f):
                return f.isfile() and '.doctrees' not in f.components()

            with closing(zipfile.ZipFile(zipfile_path, 'w')) as zf:
                for f in html_build_dir.walk(filter=zipfile_inclusion_filter):
                    zf.write(f, html_build_dir.rel_path_to(f))

            #
            # Rebuild the imported document list and search index.
            #
            if not kwargs['reindex']:
                continue

            if verbosity >= 2:
                print " reindexing..."

            # Build a dict of {path_fragment: document_object}. We'll pop values
            # out of this dict as we go which'll make sure we know which
            # remaining documents need to be deleted (and unindexed) later on.
            documents = dict((doc.path, doc) for doc in release.documents.all())

            # Walk the tree we've just built looking for ".fjson" documents
            # (just JSON, but Sphinx names them weirdly). Each one of those
            # documents gets a corresponding Document object created which
            # we'll then ask Sphinx to reindex.
            #
            # We have to be a bit careful to reverse-engineer the correct
            # relative path component, especially for "index" documents,
            # otherwise the search results will be incorrect.
            json_build_dir = destdir.child('_build', 'json')
            for built_doc in json_build_dir.walk():
                if built_doc.isfile() and built_doc.ext == '.fjson':

                    # Convert the built_doc path which is now an absolute
                    # path (i.e. "/home/docs/en/1.2/_build/ref/models.json")
                    # into a path component (i.e. "ref/models").
                    path = json_build_dir.rel_path_to(built_doc)
                    if path.stem == 'index':
                        path = path.parent
                    path = str(path.parent.child(path.stem))

                    # Read out the content and create a new Document object for
                    # it. We'll strip the HTML tags here (for want of a better
                    # place to do it).
                    with open(built_doc) as fp:
                        json_doc = json.load(fp)
                        try:
                            json_doc['body'] # Just to make sure it exists.
                            title = unescape_entities(strip_tags(json_doc['title']))
                        except KeyError, ex:
                            if verbosity >= 2:
                                print "Skipping: %s (no %s)" % (path, ex.args[0])
                            continue

                    doc = documents.pop(path, Document(path=path, release=release))
                    doc.title = title
                    doc.save()
                    haystack.site.update_object(doc)

            # Clean up any remaining documents.
            for doc in documents.values():
                if verbosity >= 2:
                    print "Deleting:", doc
                haystack.site.remove_object(doc)
                doc.delete()

    def update_svn(self, url, destdir):
        subprocess.call(['svn', 'checkout', '-q', url, destdir])

    def update_git(self, url, destdir):
        if '@' in url:
            repo, branch = url.rsplit('@', 1)
        else:
            repo, branch = url, 'master'
        if destdir.child('.git').exists():
            try:
                cwd = os.getcwdu()
                os.chdir(destdir)
                subprocess.call(['git', 'reset', '--hard', 'HEAD'])
                subprocess.call(['git', 'pull'])
            finally:
                os.chdir(cwd)
        else:
            subprocess.call(['git', 'clone', '-q', '--branch', branch, repo, destdir])
Something went wrong with that request. Please try again.