Merge pull request #666 from open-contracting/build

Better document the build

.babel_codelists

@@ -0,0 +1,2 @@
+[codelists_text: standard/schema/codelists/*.csv]
+[codelists_text: standard/docs/en/extensions/codelists/*.csv]

.babel_schema

@@ -0,0 +1 @@
+[jsonschema_text: standard/schema/*.json]

.gitignore

@@ -1,11 +1,15 @@
-.ropeproject
-*.swp
-*.pyc
 *.ipynb_checkpoints
-venv
+*.mo
+*.pyc
+*.swp
+.cache
+.ropeproject
 .ve
+venv
 
-/standard/schema/codelists_translated
-/standard/docs/en/extensions/codelists_translated
+/*.egg-info
 /build
-*.mo
+/chromedriver
+/chromedriver_linux64.zip
+/chromedriver_mac64.zip
+/chromedriver_win32.zip

.travis.yml

@@ -1,24 +1,25 @@
 sudo: false
 language: python
+cache: pip
 python:
-  - "3.5"
+  - "3.6"
 addons:
   apt:
     packages:
     - lftp
   chrome: stable
-# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
-install:
-  - pip install -r requirements.txt
-  - "curl -O https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip; unzip chromedriver_linux64.zip -d chromedriver; export PATH=$PATH:`pwd`/chromedriver"
-# command to run tests, e.g. python setup.py test
-script:
-  - flake8
-  - ./build_docs.sh
-  - py.test -rs --tb=line
 deploy:
-  skip_cleanup: true # don't remove stuff we've built
+  skip_cleanup: true
   provider: script
   script: curl https://raw.githubusercontent.com/OpenDataServices/opendataservices-deploy/master/open-contracting-standard-deploy.sh | bash -
   on:
     all_branches: true
+install:
+  - curl -O https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip
+  - unzip chromedriver_linux64.zip -d chromedriver
+  - export PATH=$PATH:`pwd`/chromedriver
+  - pip install -r requirements.txt
+script:
+  - flake8
+  - make
+  - py.test -rs --tb=line

Makefile

@@ -0,0 +1,4 @@
+# Update this file from a profile with:
+# curl -O https://raw.githubusercontent.com/open-contracting/standard_profile_template/master/Makefile
+
+include include/prologue.mk include/config.mk include/common.mk

common-requirements.txt

@@ -0,0 +1,14 @@
+# Update this file from a profile with:
+# curl -O https://raw.githubusercontent.com/open-contracting/standard_profile_template/master/common-requirements.txt
+
+# The following lines are updated based on the `repos:dependencies` task in standard-maintenance-scripts.
+
+-e git+https://github.com/open-contracting/documentation-support.git@15c73ba2b5fd3c7fc6dbf3a81fdba4ea3dfd4aec#egg=ocds_documentation_support
+-e git+https://github.com/open-contracting/sphinxcontrib-opencontracting.git@f193169306956806653dee8fb1b40c720a6f61f1#egg=sphinxcontrib-opencontracting
+-e git+https://github.com/OpenDataServices/sphinxcontrib-jsonschema.git@e39a92f4238bf022db11e4e88ba92fdaff2b31c9#egg=sphinxcontrib-jsonschema
+-e git+https://github.com/OpenDataServices/sphinxcontrib-opendataservices.git@fab0ff0167d32ec243d42f272e0e50766299c078#egg=sphinxcontrib-opendataservices
+
+# The following lines are stable and should follow the above lines to install the correct versions of dependencies.
+
+-e git+https://github.com/rtfd/recommonmark.git@81d7c6f7b37981ac22571dd91a7cc9d24c3e66a1#egg=recommonmark
+-e git+https://github.com/open-contracting/standard_theme.git@open_contracting#egg=standard_theme

include/common.mk

@@ -0,0 +1,123 @@
+# Update this file from a profile with:
+# curl -O https://raw.githubusercontent.com/open-contracting/standard_profile_template/master/include/common.mk
+
+# See https://github.com/datamade/data-making-guidelines
+
+# See http://clarkgrubb.com/makefile-style-guide#phony-target-arg
+FORCE:
+
+# http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
+COMMA := ,
+SPACE :=
+SPACE +=
+COMMA_SEPARATED_TRANSLATIONS=$(subst $(SPACE),$(COMMA),$(TRANSLATIONS:.%=%))
+
+# See http://clarkgrubb.com/makefile-style-guide#phony-targets
+.PHONY: clean
+clean:
+	rm -rf $(BUILD_DIR)
+	rm -rf $(EXTRA_BUILD_FILES)
+	rm -f $(LOCALE_DIR)/*/LC_MESSAGES/*.mo
+	rm -f $(LOCALE_DIR)/*/LC_MESSAGES/*/*.mo
+
+.PHONY: clean_dist
+clean_dist:
+	if [ -n "$(DIST_FILES)" ]; then bash -O extglob -c "rm -rf $(DIST_FILES)"; fi
+
+### Directories
+
+$(BUILD_DIR):
+	mkdir -p $(BUILD_DIR)
+
+$(POT_DIR):
+	mkdir -p $(POT_DIR)
+
+### Message catalogs
+
+.PHONY: extract_codelists
+extract_codelists: $(POT_DIR)
+	pybabel -q extract -F .babel_codelists . -o $(POT_DIR)/$(DOMAIN_PREFIX)codelists.pot
+
+.PHONY: extract_schema
+extract_schema: $(POT_DIR)
+	pybabel -q extract -F .babel_schema . -o $(POT_DIR)/$(DOMAIN_PREFIX)schema.pot
+
+# The codelist CSV files and JSON Schema files must be present for the `csv-table-no-translate` and `jsonschema`
+# directives to succeed, but the contents of the files have no effect on the generated .pot files.
+# See http://www.sphinx-doc.org/en/stable/builders.html#sphinx.builders.gettext.MessageCatalogBuilder
+.PHONY: extract_markdown
+extract_markdown: current_lang.en
+	sphinx-build -q -b gettext $(DOCS_DIR) $(POT_DIR)
+
+.PHONY: extract
+extract: extract_codelists extract_schema extract_markdown clean_current_lang
+
+### Transifex
+
+.PHONY: clean_txconfig
+clean_txconfig:
+	rm -f .tx/config
+	sphinx-intl create-txconfig
+
+.PHONY: update_txconfig
+update_txconfig:
+	sphinx-intl update-txconfig-resources --transifex-project-name $(TRANSIFEX_PROJECT) --pot-dir $(POT_DIR) --locale-dir $(LOCALE_DIR)
+
+# Builds and pushes the .pot files (`source_file` in .tx/config) to Transifex.
+.PHONY: push
+push: extract
+	tx push -s
+
+# Also pushes the translation .po files (`file_filter` in .tx/config) to Transifex.
+.PHONY: force_push_all
+force_push_all: extract
+	tx push -s -t -f -l $(COMMA_SEPARATED_TRANSLATIONS) --no-interactive
+
+pull.%: FORCE
+	tx pull -f -l $*
+
+.PHONY: pull
+pull:
+	tx pull -f -l $(COMMA_SEPARATED_TRANSLATIONS)
+
+### Current language
+
+# Create a symlink for the language, so that file paths in `jsonschema` directives resolve.
+# (Don't use clean_current_lang as a prerequisite, as then it won't run as a prerequisite later.)
+$(LANGUAGES:.%=current_lang.%): current_lang.%: FORCE
+	rm -f $(BUILD_DIR)/current_lang
+	rm -f $(BUILD_DIR)/codelists/current_lang
+	mkdir -p $(BUILD_DIR)/codelists
+	ln -s $* $(BUILD_DIR)/current_lang
+	ln -s $* $(BUILD_DIR)/codelists/current_lang
+
+# Deploy script complains if current_lang is present.
+.PHONY: clean_current_lang
+clean_current_lang:
+	rm $(BUILD_DIR)/current_lang
+	rm $(BUILD_DIR)/codelists/current_lang
+
+### Build
+
+# Build the source documentation.
+# See http://www.sphinx-doc.org/en/stable/builders.html#sphinx.builders.html.DirectoryHTMLBuilder
+.PHONY: build_source
+build_source: current_lang.en
+	sphinx-build -q -b dirhtml $(DOCS_DIR) $(BUILD_DIR)/en
+
+# Build the translated documentation. (Same as source, but with a language configuration setting.)
+$(TRANSLATIONS:.%=build.%): build.%: current_lang.%
+	sphinx-build -q -b dirhtml $(DOCS_DIR) $(BUILD_DIR)/$* -D language="$*"
+
+# Copy the assets into the build directory.
+.PHONY: assets
+assets: $(BUILD_DIR)
+	if [ -n "$(ASSETS_DIR)" ]; then cp -r $(ASSETS_DIR) $(BUILD_DIR); fi
+
+.PHONY: source
+source: assets | build_source clean_current_lang
+
+$(TRANSLATIONS:.%=%): %: assets | build_source compile build.% clean_current_lang
+
+.PHONY: all
+all: assets | build_source compile $(TRANSLATIONS:.%=build.%) clean_current_lang

include/config.mk

@@ -0,0 +1,36 @@
+# Compare this file to:
+# https://github.com/open-contracting/standard_profile_template/blob/master/config.mk
+
+# Edit these variables as appropriate.
+
+# The space-separated, period-prefixed translations to build (for easier substitutions).
+TRANSLATIONS=.es .fr
+# The source language and translations to build.
+LANGUAGES=.en $(TRANSLATIONS)
+
+# Directory of documentation files to build with Sphinx.
+DOCS_DIR=standard/docs/en
+# Directory of catalog files.
+LOCALE_DIR=standard/docs/locale
+# Directory in which to build documentation files.
+BUILD_DIR=build
+# Extra build files or directories.
+EXTRA_BUILD_FILES=
+# Files that are built and distributed (you may use Bash extended globbing).
+DIST_FILES=
+# Directory in which to build .pot files.
+POT_DIR=$(BUILD_DIR)/locale
+# The prefix, if any, to the schema and codelists domains.
+DOMAIN_PREFIX=
+# Directory containing assets to copy to the build directory (no trailing slash).
+ASSETS_DIR=standard/assets
+# The Transifex project name.
+TRANSIFEX_PROJECT=open-contracting-standard-1-1
+
+# Compile PO files for codelists and schema to MO files, so that translate_codelists and translate_schema succeed.
+.PHONY: compile
+compile:
+	pybabel compile --use-fuzzy -d $(LOCALE_DIR) -D schema
+	pybabel compile --use-fuzzy -d $(LOCALE_DIR) -D codelists
+
+# Put local targets below.

include/prologue.mk

@@ -0,0 +1,10 @@
+# Update this file from a profile with:
+# curl -O https://raw.githubusercontent.com/open-contracting/standard_profile_template/master/include/prologue.mk
+
+# See http://clarkgrubb.com/makefile-style-guide#prologue
+MAKEFLAGS += --warn-undefined-variables
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+.DEFAULT_GOAL := all
+.DELETE_ON_ERROR:
+.SUFFIXES:

requirements.txt

@@ -1,46 +1,13 @@
-Sphinx==1.5.1
-sphinx-intl==0.9.9
-# recommonmark commit pinned from master branch
--e git+https://github.com/rtfd/recommonmark.git@81d7c6f7b37981ac22571dd91a7cc9d24c3e66a1#egg=recommonmark
-flake8==3.3.0
-lxml==4.1.0
-pytest==2.8.7
-requests-mock==0.7.0
-requests==2.9.1
+-r common-requirements.txt
+
+# Add your own requirements below.
+
+# Tests
 jsonschema==2.5.1
-jsonpointer==1.10
--e git+https://github.com/open-contracting/standard_theme.git@open_contracting#egg=standard_theme
--e git+https://github.com/OpenDataServices/sphinxcontrib-jsonschema.git@09ddae1fe1aa1d8c2a56d1c937727e715074d5f8#egg=sphinxcontrib-jsonschema
--e git+https://github.com/open-contracting/ocds_sphinx_directives.git@199ad443e056b331ee489a59621eaeaf7a590340#egg=ocds_sphinx_directives
--e git+https://github.com/OpenDataServices/sphinxcontrib-opendataservices.git@23ce17656feaa237584af8822bd57ac39b498f93#egg=sphinxcontrib-opendataservices
-transifex-client==0.12.5
-json-merge-patch==0.1
+path.py==8.1.2
+pytest==2.8.7
+  py==1.4.31
 selenium==3.5.0
-.
 
-## Dependencies of the above
-alabaster==0.7.7
-Babel==2.2.0
-click==6.3
-CommonMark==0.5.4
-docutils==0.12
-et-xmlfile==1.0.1
-imagesize==0.7.1
-jdcal==1.3
-Jinja2==2.8
-jsonref==0.1
-Markdown==2.6.5
-MarkupSafe==0.23
-mccabe==0.6.1
-openpyxl==2.3.5
-path.py==8.1.2
-polib==1.0.7
-py==1.4.31
-pycodestyle==2.3.1
-pyflakes==1.5.0
-Pygments==2.1.1
-pytz==2015.7
-schema==0.6.6
-six==1.10.0
-snowballstemmer==1.2.1
-urllib3==1.22
+# Utils
+lxml