Better document the build

.babel_codelists

@@ -0,0 +1,2 @@
+[codelists_text: standard/schema/codelists/*.csv]
+[codelists_text: standard/docs/en/extensions/codelists/*.csv]

.babel_schema

@@ -0,0 +1 @@
+[jsonschema_text: standard/schema/*.json]

.gitignore

@@ -5,7 +5,5 @@
 venv
 .ve
 
-/standard/schema/codelists_translated
-/standard/docs/en/extensions/codelists_translated
 /build
 *.mo

.travis.yml

@@ -14,11 +14,11 @@ install:
 # command to run tests, e.g. python setup.py test
 script:
   - flake8
-  - ./build_docs.sh
+  - make
   - py.test -rs --tb=line
 deploy:
   skip_cleanup: true # don't remove stuff we've built
   provider: script
-  script: curl https://raw.githubusercontent.com/OpenDataServices/opendataservices-deploy/master/open-contracting-standard-deploy.sh | bash -
+  script: curl https://raw.githubusercontent.com/jpmckinney/opendataservices-deploy/exclude/open-contracting-standard-deploy.sh | bash -
   on:
     all_branches: true

Makefile

@@ -0,0 +1,141 @@
+# See https://github.com/datamade/data-making-guidelines
+
+# See http://clarkgrubb.com/makefile-style-guide#prologue
+MAKEFLAGS += --warn-undefined-variables
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+.DEFAULT_GOAL := all
+.DELETE_ON_ERROR:
+.SUFFIXES:
+
+# See http://clarkgrubb.com/makefile-style-guide#phony-target-arg
+FORCE:
+
+# The space-separated, period-prefixed translations to build (for easier substitutions).
+TRANSLATIONS=.es .fr
+# The source language and translations to build.
+LANGUAGES=.en $(TRANSLATIONS)
+
+# Directory of documentation files to build with Sphinx.
+DOCS_DIR=standard/docs/en
+# Directory of catalog files.
+CATALOGS_DIR=standard/docs/locale
+# Directory in which to build documentation files.
+BUILD_DIR=build
+# Directory in which to build .pot files.
+LOCALE_DIR=$(BUILD_DIR)/locale
+
+# http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
+COMMA := ,
+SPACE :=
+SPACE +=
+COMMA_SEPARATED_TRANSLATIONS=$(subst $(SPACE),$(COMMA),$(TRANSLATIONS:.%=%))
+
+# See http://clarkgrubb.com/makefile-style-guide#phony-targets
+.PHONY: clean
+clean:
+	rm -rf $(BUILD_DIR)
+	rm -f $(CATALOGS_DIR)/*/LC_MESSAGES/*.mo
+	rm -f $(CATALOGS_DIR)/*/LC_MESSAGES/*/*.mo
+
+### Message catalogs
+
+$(LOCALE_DIR):
+	mkdir -p $(LOCALE_DIR)
+
+.PHONY: extract_codelists
+extract_codelists: $(LOCALE_DIR)
+	pybabel -q extract -F .babel_codelists . -o $(LOCALE_DIR)/codelists.pot
+
+.PHONY: extract_schema
+extract_schema: $(LOCALE_DIR)
+	pybabel -q extract -F .babel_schema . -o $(LOCALE_DIR)/schema.pot
+
+# The codelist CSV files and JSON Schema files must be present for the `csv-table-no-translate` and `jsonschema`
+# directives to succeed, but the contents of the files have no effect on the generated .pot files.
+# See http://www.sphinx-doc.org/en/stable/builders.html#sphinx.builders.gettext.MessageCatalogBuilder
+.PHONY: extract_markdown
+extract_markdown: prebuild.en
+	sphinx-build -q -b gettext $(DOCS_DIR) $(LOCALE_DIR)
+
+.PHONY: extract
+extract: extract_codelists extract_schema extract_markdown clean_current_lang
+
+### Transifex
+
+# Builds and pushes the .pot files (`source_file` in .tx/config) to Transifex.
+.PHONY: push
+push: extract
+	tx push -s
+
+# Also pushes the translation .po files (`file_filter` in .tx/config) to Transifex.
+.PHONY: force_push_all
+force_push_all: extract
+	tx push -s -t -l $(COMMA_SEPARATED_TRANSLATIONS) -f --no-interactive
+
+pull.%: FORCE
+	tx pull -l $* -f
+
+.PHONY: pull
+pull:
+	tx pull -l $(COMMA_SEPARATED_TRANSLATIONS) -f
+
+### Pre-build
+
+# Create translated codelist CSV files, referenced by `csv-table-no-translate` directives.
+translate_codelists.%: FORCE
+	python standard/schema/utils/translate_codelists.py standard/schema/codelists $(BUILD_DIR)/codelists $(CATALOGS_DIR) $*
+	python standard/schema/utils/translate_codelists.py standard/docs/en/extensions/codelists $(BUILD_DIR)/codelists $(CATALOGS_DIR) $*
+
+# Create translated JSON Schema files in the language's build directory, referenced by `jsonschema` directives.
+translate_schema.%: FORCE
+	python standard/schema/utils/translate_schema.py standard/schema $(BUILD_DIR) $(CATALOGS_DIR) $*
+
+# Create a symlink for the language, so that file paths in `jsonschema` directives resolve.
+# (Don't use clean_current_lang as a prerequisite, as then it won't run as a prerequisite later.)
+current_lang.%: FORCE
+	rm -f $(BUILD_DIR)/current_lang
+	rm -f $(BUILD_DIR)/codelists/current_lang
+	ln -s $* $(BUILD_DIR)/current_lang
+	ln -s $* $(BUILD_DIR)/codelists/current_lang
+
+$(LANGUAGES:.%=prebuild.%): prebuild.%: translate_codelists.% translate_schema.% current_lang.%
+
+### Build
+
+# Build the source documentation.
+# See http://www.sphinx-doc.org/en/stable/builders.html#sphinx.builders.html.DirectoryHTMLBuilder
+.PHONY: build_source
+build_source: prebuild.en
+	sphinx-build -q -b dirhtml $(DOCS_DIR) $(BUILD_DIR)/en
+
+# Build the translated documentation. (Same as source, but with a language configuration setting.)
+$(TRANSLATIONS:.%=build.%): build.%: prebuild.%
+	sphinx-build -q -b dirhtml $(DOCS_DIR) $(BUILD_DIR)/$* -D language="$*"
+
+# Deploy script complains if current_lang is present.
+.PHONY: clean_current_lang
+clean_current_lang:
+	rm $(BUILD_DIR)/current_lang
+	rm $(BUILD_DIR)/codelists/current_lang
+
+# Copy the assets into the build directory.
+.PHONY: assets
+assets:
+	mkdir -p $(BUILD_DIR)
+	cp -r standard/assets $(BUILD_DIR)
+
+# Compile catalogs 'codelists.po' to 'codelists.mo' and 'schema.po' to 'schema.mo', so that translate_codelists.py and
+# translate_schema.py can succeed for translations.
+.PHONY: compile
+compile:
+	pybabel compile --use-fuzzy -d $(CATALOGS_DIR) -D codelists
+	pybabel compile --use-fuzzy -d $(CATALOGS_DIR) -D schema
+
+.PHONY: source
+source: assets | build_source clean_current_lang
+
+$(TRANSLATIONS:.%=%): %: assets | build_source compile build.% clean_current_lang
+
+.PHONY: all
+all: assets | build_source compile $(TRANSLATIONS:.%=build.%) clean_current_lang

requirements.txt

@@ -1,46 +1,49 @@
-Sphinx==1.5.1
+# The dependency trees are determined by `pipdeptree -fl`
+
+# Sphinx
+
 sphinx-intl==0.9.9
-# recommonmark commit pinned from master branch
--e git+https://github.com/rtfd/recommonmark.git@81d7c6f7b37981ac22571dd91a7cc9d24c3e66a1#egg=recommonmark
+  Babel==2.2.0
+    pytz==2015.7
+  click==6.3
+  setuptools==38.4.0
+  six==1.10.0
+  Sphinx==1.5.1
+    alabaster==0.7.7
+    docutils==0.12
+    imagesize==0.7.1
+    Jinja2==2.8
+      MarkupSafe==0.23
+    Pygments==2.1.1
+    requests==2.9.1
+    snowballstemmer==1.2.1
+-e git+https://github.com/OpenDataServices/sphinxcontrib-jsonschema.git@e39a92f4238bf022db11e4e88ba92fdaff2b31c9#egg=sphinxcontrib-jsonschema
+  jsonpointer==1.10
+  jsonref==0.1
+  -e git+https://github.com/rtfd/recommonmark.git@81d7c6f7b37981ac22571dd91a7cc9d24c3e66a1#egg=recommonmark
+    CommonMark==0.5.4
+-e git+https://github.com/open-contracting/ocds_sphinx_directives.git@199ad443e056b331ee489a59621eaeaf7a590340#egg=ocds_sphinx_directives
+-e git+https://github.com/OpenDataServices/sphinxcontrib-opendataservices.git@fab0ff0167d32ec243d42f272e0e50766299c078#egg=sphinxcontrib-opendataservices
+-e git+https://github.com/open-contracting/standard_theme.git@open_contracting#egg=standard_theme
+-e .
+
+# Transifex (not pinned, as not part of the build)
+
+transifex-client
+
+# Utils (not pinned, as not part of the build)
+
+json-merge-patch
+lxml
+
+# Tests
+
 flake8==3.3.0
-lxml==4.1.0
-pytest==2.8.7
-requests-mock==0.7.0
-requests==2.9.1
+  mccabe==0.6.1
+  pycodestyle==2.3.1
+  pyflakes==1.5.0
 jsonschema==2.5.1
-jsonpointer==1.10
--e git+https://github.com/open-contracting/standard_theme.git@open_contracting#egg=standard_theme
--e git+https://github.com/OpenDataServices/sphinxcontrib-jsonschema.git@09ddae1fe1aa1d8c2a56d1c937727e715074d5f8#egg=sphinxcontrib-jsonschema
--e git+https://github.com/open-contracting/ocds_sphinx_directives.git@199ad443e056b331ee489a59621eaeaf7a590340#egg=ocds_sphinx_directives
--e git+https://github.com/OpenDataServices/sphinxcontrib-opendataservices.git@23ce17656feaa237584af8822bd57ac39b498f93#egg=sphinxcontrib-opendataservices
-transifex-client==0.12.5
-json-merge-patch==0.1
-selenium==3.5.0
-.
-
-## Dependencies of the above
-alabaster==0.7.7
-Babel==2.2.0
-click==6.3
-CommonMark==0.5.4
-docutils==0.12
-et-xmlfile==1.0.1
-imagesize==0.7.1
-jdcal==1.3
-Jinja2==2.8
-jsonref==0.1
-Markdown==2.6.5
-MarkupSafe==0.23
-mccabe==0.6.1
-openpyxl==2.3.5
 path.py==8.1.2
-polib==1.0.7
-py==1.4.31
-pycodestyle==2.3.1
-pyflakes==1.5.0
-Pygments==2.1.1
-pytz==2015.7
-schema==0.6.6
-six==1.10.0
-snowballstemmer==1.2.1
-urllib3==1.22
+pytest==2.8.7
+  py==1.4.31
+selenium==3.5.0

standard/docs/en/examples/merging/merge.py

@@ -1,8 +1,9 @@
-import os
 import json
-import ocdsmerge
+import os
 from collections import OrderedDict
 
+import ocdsmerge
+
 releases = []
 release_list = []
 packages = []

standard/docs/en/extensions/bids.md

@@ -48,7 +48,7 @@ For example, publishers may wish to add statistics on minority or women owned bu
 ```eval_rst
 .. csv-table-no-translate::
    :header-rows: 1
-   :file: codelists_translated/bidStatistics.csv
+   :file: ../../../../build/codelists/current_lang/bidStatistics.csv
 ```
 
 ### Bid Details
@@ -66,7 +66,7 @@ The `bids/details` array is used to provide one or more `Bid` objects, each repr
 ```eval_rst
 .. csv-table-no-translate::
    :header-rows: 1
-   :file: codelists_translated/bidStatus.csv
+   :file: ../../../../build/codelists/current_lang/bidStatus.csv
 ```
 
 ### Example