Improve image building (primarily for discourse_dev)

Move "discourse" user and "/var/www" directory creation into base image
so that discourse_dev can build from base image directly instead of
requiring the overhead of the full discourse image.

Remove discourse_fast_switch dependency from discourse_dev, and remove
the ruby 2.2/2.0 logic, as Discourse already appears to require 2.3.0 or
greater.

Remove discourse_dev's independent `postgres.template.yml` and
`redis.template.yml` files, and instead derive them from the top-level
templates directory.  This ensures that the discourse_dev image is
always up-to-date with the discourse main image.

Add password-less "sudo" permissions for the "discourse" user in the
discourse_dev image.  This simplifies (fixes) the "bundle install"
command (and others) as implemented by discourse/bin/docker utilities.

Add image/Makefile so that images can be built even on machines without
Ruby installed.  Logic is replicated from `image/build.rb`, and should
result in equivalent images.  (Perhaps build.rb should be removed?)

Add automation in Makefile to ensure that all Dockerfiles are updated to
match the version number in the Makefile.  (This includes the `/VERSION`
file written into the base image.)

Add image/README.md to describe the images and process.

Update ImageMagick to 6.9.5-9 in the base Dockerfile because 6.9.5-8 is
no longer available. (Perhaps this dependency should be vendored.)

Fix base "boot" failure caused by missing log files on a new instance
(in `base/runit-1.d-00-fix-var-logs`).  Before chowning the log files,
"touch" them to ensure they exist.  This failure previously prevented
the discourse_dev `ensure-database` script from running, which made
development use of discourse_dev very difficult.

Update to version 1.3.7.  (Driven by IMAGE_VERSION in the Makefile.
Mostly, this is to ensure that the next official build of the images
will be *at least* 1.3.7, superseding any previous versions.)

.gitignore

@@ -3,6 +3,12 @@ shared/*
 containers/*
 cids/*
 bin/*
+image/.build.out
+image/img.tar
+image/squash.tar
 image/nsenter/nsenter
 image/docker-squash
+image/docker-squash.tar.gz
+image/discourse_dev/postgres.template.yml
+image/discourse_dev/redis.template.yml
 .gc-state/*

README.md

@@ -36,13 +36,9 @@ Placeholder spot for shared volumes with various Discourse containers. You may e
 
 #### `/image`
 
-Dockerfile for both the base image `/discourse_base` and discourse image `/discourse`.
+Dockerfiles for Discourse; see [the README](image/README.md) for further details.
 
-- `/discourse_base` contains all the OS dependencies including runit, postgres, nginx, ruby.
-
-- `/discourse` builds on the base image and configures a discourse user and `/var/www/discourse` directory for the Discourse source.
-
-The Docker repository will always contain the latest built version at: https://index.docker.io/u/samsaffron/discourse/ , you should not need to build the base image.
+The Docker repository will always contain the latest built version at: https://hub.docker.com/r/discourse/discourse/, you should not need to build the base image.
 
 ### Launcher
 

image/Makefile

@@ -0,0 +1,76 @@
+IMAGE_VERSION := 1.3.7
+
+SHELL         := /bin/bash
+OS            := $(shell uname | tr '[:upper:]' '[:lower:]')
+
+DOCKER_SQUASH_URL := https://github.com/jwilder/docker-squash/releases/download/v0.2.0/docker-squash-${OS}-amd64-v0.2.0.tar.gz
+
+
+# omitting discourse_fast_switch from 'all' as it seems obsolete.
+all: base discourse discourse_dev discourse_test discourse_bench
+.PHONY: all base discourse discourse_dev discourse_test discourse_bench discourse_fast_switch
+
+base: base-deps
+# discourse: discourse-deps
+discourse_dev: discourse_dev-deps
+# discourse_test: discourse_test-deps
+# discourse_bench: discourse_bench-deps
+# discourse_fast_switch: discourse_fast_switch-deps
+
+base:
+	@IMAGE_DIR=$@ IS_BASE=YES ${MAKE} build-image
+
+discourse discourse_fast_switch discourse_dev discourse_test discourse_bench:
+	@IMAGE_DIR=$@ ${MAKE} build-image
+
+
+base-deps: base/phantomjs
+.PHONY: base-deps
+
+base/phantomjs:
+	@echo "Fetching phantomjs..."
+	cd base && ./download_phantomjs
+
+discourse_dev-deps:
+	@echo "Copying postgres and redis templates..."
+	sed -e 's/\(db_name: discourse\)/\1_development/' ../templates/postgres.template.yml > discourse_dev/postgres.template.yml
+	cp ../templates/redis.template.yml discourse_dev/redis.template.yml
+.PHONY: discourse_dev-deps
+
+# docker-squash doesn't seem to work on OSX... there's an error about calling
+# tar with an unsuported "--xattrs" option.  If/when that gets fixed, the ifeq
+# can be removed.  For now, OSX skips the docker-squash step, so don't push
+# images built on OSX!
+build-image: docker-squash update-dockerfile
+	@echo "----- building image: discourse/${IMAGE_DIR}:${IMAGE_VERSION} -----"
+ifeq ($(OS), darwin)
+	docker build -t discourse/${IMAGE_DIR}:${IMAGE_VERSION} ${IMAGE_DIR}
+else
+	docker build ${IMAGE_DIR} | tee .build.out
+	@echo "squashing $(shell tail -1 .build.out | awk '/^Successfully built / {print $$3}')..."
+	docker save -o img.tar $(shell tail -1 .build.out | awk '/^Successfully built / {print $$3}')
+	./docker-squash -verbose -i img.tar -o squash.tar $(if $(IS_BASE),-from root) -t discourse/${IMAGE_DIR}:${IMAGE_VERSION}
+	docker load -i squash.tar
+	rm -f squash.tar
+	rm -f img.tar
+	rm -f .build.out
+endif
+.PHONY: build-image
+
+docker-squash:
+	wget ${DOCKER_SQUASH_URL} --output-document=$@.tar.gz
+	tar -xzvf $@.tar.gz
+	rm -f $@.tar.gz
+
+# We want to ensure that the Dockerfiles all have an appropriate header
+# comment, and any FROM line refers to the version we're currently building.
+# Also, if there's a line that echoes into VERSION, replace the value with the
+# current version.  (Dockerfiles support environment expansion, but not for
+# FROM or RUN statements.)
+update-dockerfile:
+	@echo "----- updating ${IMAGE_DIR}/Dockerfile for ${IMAGE_VERSION} -----"
+	@sed -i '' -e 's/^\(# NAME:\).*$$/\1     discourse\/${IMAGE_DIR}/' ${IMAGE_DIR}/Dockerfile
+	@sed -i '' -e 's/^\(# VERSION:\).*$$/\1  ${IMAGE_VERSION}/' ${IMAGE_DIR}/Dockerfile
+	@sed -i '' -e 's/^\(FROM discourse\/[^:]*:\).*/\1${IMAGE_VERSION}/' ${IMAGE_DIR}/Dockerfile
+	@sed -i '' -e 's/^\(RUN echo "\)[^"]*\(" > \/VERSION\)$$/\1${IMAGE_VERSION}\2/' ${IMAGE_DIR}/Dockerfile
+.PHONY: update-dockerfile

image/README.md

@@ -0,0 +1,45 @@
+# Docker images
+
+## Building new images
+
+To build a new set of images, update the `Makefile` with the new version number, and then `make all`.  This will automatically update the header comments in the Dockerfiles and update any `FROM` statements to ensure that the image verions remain in lock-step with each other.  (The downside is that if you only wanted to tweak a "leaf" image, you'll still be touching/updating _all_ of the images.  But reasoning about the images is much easier if they all have the same version.)
+
+> _A note about building on OSX:_ While the `Makefile` has provisions for acquiring an OS-specific version of docker-squash, running the Darwin (OSX) version doesn't work on my machine.  To cope, OSX builds simply skip the docker-squash step.  Since I'm not going to be pushing images up to Docker Hub, that's okay with me.
+
+The build process will tag the images with the version number, but not with "latest", nor will it push the images up to Docker Hub.  Both of those steps must be performed manually.
+
+## More about the images
+
+See both `Makefile` and the respective `Dockerfile`s for details on _how_ all of this happens.
+
+
+### base ([discourse/base](https://hub.docker.com/r/discourse/base/))
+
+All of the dependencies for running Discourse.  This includes runit, postgres, nginx, ruby, imagemagick, etc.  It also includes the creation of the "discourse" user and `/var/www` directory.
+
+
+### discourse ([discourse/discourse](https://hub.docker.com/r/discourse/discourse/))
+
+Builds on the base image and adds the current (as of image build time) version of Discourse, cloned from GitHub, and also the bundled gems.
+
+
+### discourse_dev ([discourse/discourse_dev](https://hub.docker.com/r/discourse/discourse_dev/))
+
+Adds redis and postgres just like the "standalone" template for Discourse in order to have an all-in-one container for development.  Note that you are expected to mount your local discourse source directory to `/src`.  See [the README in GitHub's discourse/bin/docker](https://github.com/discourse/discourse/tree/master/bin/docker/) for utilities that help with this.
+
+Note that the discourse user is granted "sudo" permission without asking for a password in the discourse_dev image.  This is to facilitate the command-line Docker tools in discourse proper that run commands as the discourse user.
+
+
+### discourse_test ([discourse/discourse_test](https://hub.docker.com/r/discourse/discourse_test/))
+
+Builds on the discourse image and adds testing tools and a default testing entrypoint.
+
+
+### discourse_bench ([discourse/discourse_bench](https://hub.docker.com/r/discourse/discourse_bench/))
+
+Builds on the discourse_test image and adds benchmark testing.
+
+
+### discourse_fast_switch ([discourse/discourse_fast_switch](https://hub.docker.com/r/discourse/discourse_fast_switch/))
+
+Builds on the discourse image and adds the ability to easily switch versions of Ruby.

image/base/Dockerfile

@@ -1,14 +1,13 @@
 # NAME:     discourse/base
-# VERSION:  1.3.6
-
+# VERSION:  1.3.7
 FROM ubuntu:16.04
 
 ENV PG_MAJOR 9.5
 ENV PG_VERSION 9.5.4-1.pgdg16.04+1
 
 MAINTAINER Sam Saffron "https://twitter.com/samsaffron"
 
-RUN echo "1.3.5" > /VERSION
+RUN echo "1.3.7" > /VERSION
 
 RUN apt-get update && apt-get install -y lsb-release sudo curl
 RUN echo "debconf debconf/frontend select Teletype" | debconf-set-selections
@@ -121,3 +120,8 @@ ADD boot /sbin/boot
 ADD cron /etc/service/cron/run
 ADD rsyslog /etc/service/rsyslog/run
 ADD cron.d_anacron /etc/cron.d/anacron
+
+# Create discourse user and /var/www location for both
+# discourse and dev images.
+RUN useradd discourse -s /bin/bash -m -U &&\
+    mkdir -p /var/www

image/base/install-imagemagick

@@ -19,7 +19,7 @@ make all && make install
 
 
 # Build and install ImageMagick
-wget -O $WDIR/ImageMagick.tar.gz "http://www.imagemagick.org/download/ImageMagick-6.9.5-8.tar.gz"
+wget -O $WDIR/ImageMagick.tar.gz "http://www.imagemagick.org/download/ImageMagick-6.9.5-9.tar.gz"
 IMDIR=$WDIR/$(tar tzf $WDIR/ImageMagick.tar.gz --wildcards "ImageMagick-*/configure" |cut -d/ -f1)
 tar zxf $WDIR/ImageMagick.tar.gz -C $WDIR
 cd $IMDIR

image/base/runit-1.d-00-fix-var-logs

@@ -1,7 +1,6 @@
 #!/bin/bash
 mkdir -p /var/log/nginx
 chown -R www-data:www-data /var/log/nginx
-chown www-data:www-data /var/log/nginx
-chown -f syslog:adm /var/log/syslog*
-chown -f syslog:adm /var/log/auth.log*
-chown -f syslog:adm /var/log/kern.log*
+touch /var/log/syslog   && chown -f syslog:adm /var/log/syslog*
+touch /var/log/auth.log && chown -f syslog:adm /var/log/auth.log*
+touch /var/log/kern.log && chown -f syslog:adm /var/log/kern.log*

image/discourse/Dockerfile

@@ -1,11 +1,11 @@
-# Current version 1.3.6
-FROM discourse/base:1.3.6
+# NAME:     discourse/discourse
+# VERSION:  1.3.7
+FROM discourse/base:1.3.7
 
 MAINTAINER Sam Saffron "https://twitter.com/samsaffron"
 
 # Discourse specific bits
-RUN useradd discourse -s /bin/bash -m -U &&\
-    mkdir -p /var/www && cd /var/www &&\
+RUN  cd /var/www &&\
      git clone https://github.com/discourse/discourse.git &&\
      cd discourse &&\
       git remote set-branches --add origin tests-passed &&\
@@ -14,4 +14,3 @@ RUN useradd discourse -s /bin/bash -m -U &&\
      sudo -u discourse bundle install --deployment \
          --without test --without development &&\
     find /var/www/discourse/vendor/bundle -name tmp -type d -exec rm -rf {} +
-

image/discourse_bench/Dockerfile

@@ -1,4 +1,6 @@
-FROM samsaffron/discourse_test:latest
+# NAME:     discourse/discourse_bench
+# VERSION:  1.3.7
+FROM discourse/discourse_test:1.3.7
 
 MAINTAINER Gerhard Schlager <mail@gerhard-schlager.at>
 
@@ -17,11 +19,12 @@ RUN gem install facter &&\
 
 # reconfigure PostgreSQL template databases to use UTF-8
 # otherwise rake db:create fails
-RUN pg_dropcluster --stop 9.3 main &&\
-    pg_createcluster --start -e UTF-8 9.3 main
+RUN pg_dropcluster --stop 9.5 main &&\
+    pg_createcluster --start -e UTF-8 9.5 main
 
 # create role "discourse"
 RUN /etc/init.d/postgresql start &&\
+    sleep 10 &&\
     sudo -u postgres createuser --superuser discourse
 
 ADD run_bench.sh /tmp/run_bench.sh

image/discourse_dev/Dockerfile

@@ -1,21 +1,11 @@
-FROM samsaffron/discourse_fast_switch:1.0.13
+# NAME:     discourse/discourse_dev
+# VERSION:  1.3.7
+FROM discourse/base:1.3.7
 
 MAINTAINER Sam Saffron "https://twitter.com/samsaffron"
 
-RUN use_22
-RUN rm -fr /usr/local/include/ruby-2.0.0
-
-RUN chown -R discourse:discourse /usr/ruby_22
-
-RUN rm /var/www/discourse/.bundle/config
-
-RUN cd /var/www/discourse && git pull && sudo -u discourse bundle install
-RUN use_22
-
-# CLEANUP
-RUN rm -fr /var/www/discourse/vendor/bundle/ruby
-RUN rm -fr /usr/ruby_20
-
+# Give discourse user no-passwd sudo permissions (for bundle install)
+ADD sudoers.discourse /etc/sudoers.d/discourse
 
 # get redis going
 ADD redis.template.yml /pups/redis.yml
@@ -30,10 +20,9 @@ ENV LC_ALL en_US.UTF-8
 ADD postgres.template.yml /pups/postgres.yml
 RUN LANG=en_US.UTF-8 /pups/bin/pups /pups/postgres.yml
 
-# move pg out of the way
+# move default postgres_data out of the way
 RUN mv /shared/postgres_data /shared/postgres_data_orig
 
-# We got gems ... we need no source in the image ...
-RUN rm -fr /var/www/discourse
-
+# re-instantiate data on boot if needed (this will allow it to persist across
+# invocations when used with a mounted volume)
 ADD ensure-database /etc/runit/1.d/ensure-database

image/discourse_dev/ensure-database

@@ -2,7 +2,7 @@
 
 if [ ! -f /shared/postgres_data/PG_VERSION ]; then
   mkdir -p /shared/postgres_data
-  cp -R /shared/postgres_data_orig/* /shared/postgres_data
   chown -R postgres:postgres /shared/postgres_data
   chmod 700 /shared/postgres_data
+  cp -R /shared/postgres_data_orig/* /shared/postgres_data
 fi