diff --git a/.travis.yml b/.travis.yml index cee3f3e..c7ef510 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,5 +8,5 @@ addons: - shellcheck script: - - shellcheck build.sh - - shellcheck lint.sh + - shellcheck *.sh + - shellcheck ci-deploy/*.sh diff --git a/build.sh b/build.sh index 7cc4a7f..b15efbb 100755 --- a/build.sh +++ b/build.sh @@ -3,7 +3,7 @@ set -e HTML_GIT_CLONE_OPTIONS=${HTML_GIT_CLONE_OPTIONS:-"--depth 1"} # cd to the directory containing this script -cd "$( dirname "${BASH_SOURCE[0]}" )" +cd "$(dirname "$0")" DIR=$(pwd) DO_UPDATE=true diff --git a/ci-deploy/.dockerignore b/ci-deploy/.dockerignore new file mode 100644 index 0000000..eb41cad --- /dev/null +++ b/ci-deploy/.dockerignore @@ -0,0 +1,3 @@ +# Git checkout metadata changes every time you check out. If we left these directories there, Docker's caches (based on +# comparing the contents of the files ADDed) would be invalidated. +**/.git diff --git a/ci-deploy/Dockerfile b/ci-deploy/Dockerfile new file mode 100644 index 0000000..d061c9b --- /dev/null +++ b/ci-deploy/Dockerfile @@ -0,0 +1,25 @@ +# This Dockerfile is just used to run on Travis CI in an environment that can easily and repeatedly +# install our build dependencies. +FROM debian:sid + +RUN apt-get update && \ + apt-get install -y ca-certificates curl rsync git unzip fp-compiler default-jre + +ADD wattsi /whatwg/wattsi + +RUN cd /whatwg/wattsi && \ + /whatwg/wattsi/build.sh +ENV PATH="/whatwg/wattsi/bin:${PATH}" + +ADD html-build /whatwg/html-build + +# Note: we do not ADD /whatwg/html, but instead mount it using --volume in .travis.yml, since it +# contains the deploy_key, and thus should not be part of the image. The image is cached, publicly, +# on Docker Hub. +ENV HTML_SOURCE /whatwg/html + +ARG travis_pull_request +ENV TRAVIS_PULL_REQUEST=${travis_pull_request} + +ENV SKIP_BUILD_UPDATE_CHECK=true +ENTRYPOINT ["bash", "/whatwg/html-build/ci-deploy/inside-container.sh"] diff --git a/ci-deploy/README.md b/ci-deploy/README.md new file mode 100644 index 0000000..d223f12 --- /dev/null +++ b/ci-deploy/README.md @@ -0,0 +1,10 @@ +# HTML Standard CI Deploy + +This directory contains files used specifically for deploying the HTML Standard on Travis CI. They are not generally relevant to local builds. + +The setup is assumed to be a directory containing: + +- A subdirectory `html-build` containing the contents of this entire [whatwg/html-build](https://github.com/whatwg/html-build) repository +- A subdirectory `html` containing the contents of the [whatwg/html](https://github.com/whatwg/html) repository + +Then, run the `html-build/ci-deploy/outside-container.sh` script. What it does is documented via inline comments; check it out to learn more. In particular, note that several environment variables are assumed to be set, via the CI system. diff --git a/ci-deploy/deploy-key.enc b/ci-deploy/deploy-key.enc new file mode 100644 index 0000000..06f68a8 Binary files /dev/null and b/ci-deploy/deploy-key.enc differ diff --git a/ci-deploy/inside-container.sh b/ci-deploy/inside-container.sh new file mode 100644 index 0000000..e804ff1 --- /dev/null +++ b/ci-deploy/inside-container.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail +cd "$(dirname "$0")/../.." + +WEB_ROOT="html.spec.whatwg.org" +DEPLOY_USER="annevankesteren" + +SERVER="75.119.197.251" +SERVER_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDP7zWfhJdjre9BHhfOtN52v6kIaDM/1kEJV4HqinvLP2hzworwNBmTtAlIMS2JJzSiE+9WcvSbSqmw7FKmNVGtvCd/CNJJkdAOEzYFBntYLf4cwNozCRmRI0O0awTaekIm03pzLO+iJm0+xmdCjIJNDW1v8B7SwXR9t4ElYNfhYD4HAT+aP+qs6CquBbOPfVdPgQMar6iDocAOQuBFBaUHJxPGMAG0qkVRJSwS4gi8VIXNbFrLCCXnwDC4REN05J7q7w90/8/Xjt0q+im2sBUxoXcHAl38ZkHeFJry/He2CiCc8YPoOAWmM8Vd0Ukc4SYZ99UfW/bxDroLHobLQ9Eh" + +HTML_SHA=$(git -C html rev-parse HEAD) + +# `export`ed because build.sh reads it +HTML_OUTPUT="$(pwd)/output" +export HTML_OUTPUT + +# Environment variables set from outside +TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:-false} + +# Build the spec into the output directory +./html-build/build.sh + +# Conformance-check the result +echo "" +echo "Downloading and running conformance checker..." +curl --remote-name --fail https://sideshowbarker.net/nightlies/jar/vnu.jar +java -jar vnu.jar --skip-non-html "$HTML_OUTPUT" +echo "" + +# Note: $TRAVIS_PULL_REQUEST is either a number or false, not true or false. +# https://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables +if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then + echo "Skipping deploy for non-master" + exit 0 +fi + +# Add the (decoded) deploy key to the SSH agent, so scp works +chmod 600 html/deploy-key +eval "$(ssh-agent -s)" +ssh-add html/deploy-key +echo "$SERVER $SERVER_PUBLIC_KEY" > known_hosts + +# Sync, including deletes, but ignoring the commit-snapshots directory so we don't delete that. +echo "Deploying build output..." +rsync --rsh="ssh -o UserKnownHostsFile=known_hosts" \ + --archive --compress --delete --verbose --exclude="commit-snapshots" \ + "$HTML_OUTPUT/" "$DEPLOY_USER@$SERVER:$WEB_ROOT" + +# Now sync a commit snapshot +# (See https://github.com/whatwg/html-build/issues/97 potential improvements to commit snapshots.) +echo "" +echo "Deploying commit snapshot..." +rsync --rsh="ssh -o UserKnownHostsFile=known_hosts" \ + --archive --compress --verbose \ + "$HTML_OUTPUT/index.html" "$DEPLOY_USER@$SERVER:$WEB_ROOT/commit-snapshots/$HTML_SHA" + +echo "" +echo "All done!" diff --git a/ci-deploy/outside-container.sh b/ci-deploy/outside-container.sh new file mode 100644 index 0000000..0a617c3 --- /dev/null +++ b/ci-deploy/outside-container.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +HERE=$(dirname "$0") +cd "$HERE/../.." + +DOCKER_USERNAME="domenicdenicola" +DOCKER_HUB_REPO="whatwg/html-deploy" +# DOCKER_PASSWORD is set from the outside +# TRAVIS_PULL_REQUEST is set from the outside +# ENCRYPTION_LABEL is set from the outside + +git clone https://github.com/whatwg/wattsi.git wattsi + +# Copy the Docker-related stuff into the working (grandparent) directory. +cp "$HERE"/{.dockerignore,Dockerfile} . + +# Build the Docker image, using Docker Hub as a cache. (This will be fast if nothing has changed +# in wattsi or html-build). +docker pull "$DOCKER_HUB_REPO:latest" +docker build --cache-from "$DOCKER_HUB_REPO:latest" \ + --tag "$DOCKER_HUB_REPO:latest" \ + --build-arg "travis_pull_request=$TRAVIS_PULL_REQUEST" \ + . + +# Decrypt the deploy key from this script's location into the html/ directory, since that's the +# directory that will be shared with the container (but not built into the image). +ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key" +ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv" +ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR} +ENCRYPTED_IV=${!ENCRYPTED_IV_VAR} +openssl aes-256-cbc -K "$ENCRYPTED_KEY" -iv "$ENCRYPTED_IV" \ + -in "$HERE/deploy-key.enc" -out html/deploy-key -d + +# Run the inside-container.sh script, with the html/ directory mounted inside the container. +echo "" +docker run --volume "$(pwd)/html":/whatwg/html "$DOCKER_HUB_REPO:latest" + +# If the build succeeded and we got here, upload the Docker image to Docker Hub, so that future runs +# can use it as a cache. +echo "" +docker tag "$DOCKER_HUB_REPO:latest" "$DOCKER_HUB_REPO:$TRAVIS_BUILD_NUMBER" && +docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" +docker push "$DOCKER_HUB_REPO"