Upgrades, secrets, and ONBUILD images (version 3.0.0) (#43)

* improve and document BOINC server development workflow

* readme tweaks

* fix headings

* fix variable

* make the files built in the local volume have non-root permissions

(and consequently make everything in Docker containers run as a non-root 
user, which has been requested for a while)

* fix accidentally updated submodule

* temp make default TAG non-b2d so demo works

* add upgrade instructions

* fix some stuff rebuilding unnecessarily

* update -b2d to work with new setup as well

* cleanup .env

* remove unnecessary command

* upgrade to debian:stretch and mariadb:10; cleanup handling of "secrets"

* minor cleanup

* remove dev workflow since it doesnt really work right yet

* use multi-stage build to remove redundant Dockerfile and Dockefile-b2d

* remove leftover dev workflow file

* make images use ONBUILD for better user customization

* update docs, add secrets section

* fix secrets volume being overwritten, update example project

.env

@@ -1,3 +1,37 @@
+# controls which TAGs to build. should be either "TAG=-b2d" to build the server with
+# boinc2docker installed, or blank ("TAG=") to build the vanilla server
+TAG=-b2d
+
+
+
+# -----------------------------------------------------------------------
+# the following can be changed even with a project running, and will be
+# applied the next time you run `docker compose up -d`
+# -----------------------------------------------------------------------
+
+# the URL the server thinks its at
 URL_BASE=http://127.0.0.1
+
+
+
+# -----------------------------------------------------------------------------
+# the following must be set before you build a project, and cannot be changed
+# afterwards
+# -----------------------------------------------------------------------------
+
+# the project name
 PROJECT=boincserver
-TAG=-b2d
+
+# the user who builds the server source and runs the BOINC daemons
+BOINC_USER=boincadm
+
+# the folder where the project is created
+PROJECT_ROOT=/home/boincadm/project
+
+
+
+# --------------------------------------------------
+# used for development, not for general modification
+# --------------------------------------------------
+DEFAULTARGS=-defaultargs
+VERSION=latest

Makefile

@@ -36,7 +36,7 @@ rm-apache:
 	$(DC) stop apache && $(DC) rm -f apache
 
 exec-apache:
-	$(DC) exec apache bash
+	$(DC) exec -u boincadm apache bash
 
 
 # --- mysql ---
@@ -53,8 +53,22 @@ rm-mysql:
 
 # --- for local building/testing ---
 
-TAG=$(shell git describe --tags --abbrev=0)
-tag:
-	docker tag boinc/server_apache:latest-b2d boinc/server_apache:$(TAG)-b2d
-	docker tag boinc/server_mysql:latest boinc/server_mysql:$(TAG)
-	docker tag boinc/server_makeproject:latest-b2d boinc/server_makeproject:$(TAG)-b2d
+build-and-tag-all:
+	for TAG in "" "-b2d"; do \
+	    for DEFAULTARGS in "" "-defaultargs"; do \
+	        for VERSION in "latest" $(shell git describe --tags --abbrev=1); do \
+	            export TAG VERSION DEFAULTARGS ; \
+	            docker-compose build 2>&1 | grep --color=never "Successfully tagged"; \
+	        done ; \
+	    done ; \
+	done
+
+push-all:
+	for TAG in "" "-b2d"; do \
+	    for DEFAULTARGS in "" "-defaultargs"; do \
+	        for VERSION in "latest" $(shell git describe --tags --abbrev=1); do \
+	            export TAG VERSION DEFAULTARGS ; \
+	            docker-compose push ; \
+	        done ; \
+	    done ; \
+	done

README.md

@@ -1,31 +1,31 @@
 
-boinc-server-docker
-===================
+# boinc-server-docker
 
 `boinc-server-docker` is the easiest way to run your own [BOINC](http://boinc.berkeley.edu/) server. You can run the server on a Linux machine, in which case the requirements are, 
 
-* [Docker](https://docs.docker.com/engine/installation/) (>=17.03.0ce)
-* [docker-compose](https://docs.docker.com/compose/install/) (>=1.13.0 but !=1.19.0 due to a [bug](https://github.com/docker/docker-py/issues/1841))
+* [Docker](https://docs.docker.com/engine/installation/) (>=17.09.0ce)
+* [docker-compose](https://docs.docker.com/compose/install/) (>=1.17.0 but !=1.19.0 due to a [bug](https://github.com/docker/docker-py/issues/1841))
 * git
 
 or you can run your server on Windows 7+ or Mac OSX, in which case you should use either,
 
-* [Docker for Mac](https://docs.docker.com/docker-for-mac/install/#download-docker-for-) (>=17.06.0ce)
-* [Docker for Windows](https://docs.docker.com/docker-for-windows/install/) (>=17.06.0ce)
+* [Docker for Mac](https://docs.docker.com/docker-for-mac/install/#download-docker-for-) (>=17.09.0ce)
+* [Docker for Windows](https://docs.docker.com/docker-for-windows/install/) (>=17.09.0ce)
 
 or, if your Windows/Mac system is too old to support either of those,
 
-* [Docker Toolbox](https://docs.docker.com/toolbox/overview) (>=17.05.0ce)
+* [Docker Toolbox](https://docs.docker.com/toolbox/overview) (>=17.09.0ce)
 
 There are no other dependencies, as everything else is packaged inside of Docker. 
 
 
-Documentation
--------------
+## Documentation
 
-For a full tutorial on setting up the server, see the [project cookbook](https://github.com/marius311/boinc-server-docker/blob/master/docs/cookbook.md). 
+For a full tutorial on creating your own server with `boinc-server-docker`, see the [project cookbook](https://github.com/marius311/boinc-server-docker/blob/master/docs/cookbook.md). 
 
-If you are somewhat familiar with Docker and BOINC, the following short description takes you through creating a server and running your own science application. 
+If you are looking to create a server and are already somewhat familiar with Docker and BOINC, the following short description takes you through creating a server and running your own science application. 
+
+### Quickstart
 
 To check out this repository and get a test server fully up and running, simply run,
 ```bash
@@ -68,7 +68,7 @@ This is a simple example, but any Docker containers with arbitrary code installe
 
 To stop the server and delete all server and database files (for example, if you want to start over with a fresh copy), run,
 
-```
+```bash
 docker-compose down -v
 ```
 
@@ -78,8 +78,34 @@ Finally, `boinc-server-docker` is not just useful to get a simple test server ru
 Happy crunching! 
 
 
-News
-----
+## News
+
+* **Version 3.0.0** - July 27, 2018
+    * Based on [server_release/0.9](https://github.com/BOINC/boinc/releases/tag/server_release%2F0.9).
+    * Upgraded to Debian Stretch, PHP 7.0.31 and MariaDB 10.3.8. 
+    * Docker requirement is now 17.09.0ce on all platforms.  
+    * Project "secrets" such as passwords and signing keys are now stored in a new volume called `secrets`, and the procedure for how to deal with them is documented [here](docs/cookbook.md#managing-secrets). 
+    * *Breaking change:* For improved security, the BOINC daemons no longer run as root, instead they run as an unprivileged user, by default named `boincadm`. 
+    * Added two new options which are congurable at build-time, `BOINC_USER` and `PROJECT_ROOT`, and fixed `PROJECT` which wasn't fully configurable before. Under the hood, the `boinc-server-docker` images now use Docker `ONBUILD` instructions to make this happen.     
+    * *Upgrade instructions:* If you don't care about the files in your project's database and project folder, you can just wipe your project clean with `docker-compose down -v` and simply start a fresh server with version `3.0.0`. If instead you want to upgrade a project you created with `boinc-server-docker v2.x.x`, you should follow these instructions:
+        1) Edit the `FROM` line in your custom Dockerfiles to source the appropriate `3.0.0` images.
+        2) Diff your `docker-compose.yml` and `.env` files against the corresponding ones in `example_project/`, and merge in changes you see (notably, add the `secrets` volume). 
+        3) Run `docker-compose build` to build updated images. 
+        3) Run `docker-compose run --rm makeproject bash` and navigate to `/home/boincadm/secrets`. This is your `secrets` volume, and you should edit the files you see here so that they contain your passwords, keys, etc... 
+        4) Bring your project down with `docker-compose down`.
+        5) Run the following to update permissions and upgrade your database: 
+
+              ```bash
+              source .env
+              eval "$(docker-compose run --rm -T makeproject cat /run/secrets/secrets.env)"
+              
+              docker-compose run --rm -u root makeproject chown -R $BOINC_USER:$BOINC_USER $PROJECT_ROOT.dst
+
+              docker-compose exec mysql mysql_upgrade
+
+              docker-compose exec mysql mysqladmin -u root password $DB_PASSWD
+              ```
+        6) Now bring your project back up with `docker-compose up -d`.
 
 * **Version 2.1.0** - May 29, 2018
     * Update boinc to [server_release/0.9](https://github.com/BOINC/boinc/releases/tag/server_release%2F0.9).
@@ -93,10 +119,10 @@ News
     * A number of improvements to boinc2docker (see [ccfe9a9](https://github.com/marius311/boinc-server-docker/commit/ccfe9a9704b9282f528565c74e07ee3be698aa0d)).
 
 
-Development and Contributing
------------------------------
+## Development and Contributing
 
-To modify and rebuild any of the `boinc-server-docker` images, you will need this git repository's submodules checked out (run `git submodule update --init --recursive`, or clone with `git clone --recursive` in the first place). Note also that currently building the images only works on Linux. 
+For using `boinc-server-docker` to work on development of the BOINC server soure code, see the [development workflow](docs/dev-workflow.md). 
 
+There is developer documentation for `boinc-server-docker` itself, but please feel free to contact the maintainers or submit Issues and Pull Requests for this repository. 
 
-Please don't hesitate to get in contact with the maintainers of this project or to submit pull requests!
+As a reminder, to modify and rebuild any of the `boinc-server-docker` images, you will need this git repository's submodules checked out (run `git submodule update --init --recursive`, or clone with `git clone --recursive` in the first place). Note also that currently building the images only works on Linux. 

docker-compose.yml

@@ -1,51 +1,58 @@
-version: "3"
+version: "3.4"
 
 volumes:
   mysql:
   project:
   results:
+  secrets:
 
 services:
   mysql:
-    image: boinc/server_mysql:latest
-    build: images/mysql
-    network_mode: bridge
+    image: boinc/server_mysql:$VERSION$DEFAULTARGS
+    build: 
+      context: images/mysql
+      target: mysql$DEFAULTARGS
     volumes:
-     - "mysql:/var/lib/mysql"
-    environment: 
-     - MYSQL_ROOT_PASSWORD=
-     - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+      - "mysql:/var/lib/mysql"
 
   makeproject:
-    image: boinc/server_makeproject:latest$TAG
+    image: boinc/server_makeproject:$VERSION$TAG$DEFAULTARGS
     build: 
       context: images/makeproject
-      dockerfile: Dockerfile$TAG
-    network_mode: bridge
-    links: 
-     - mysql
+      target: makeproject$DEFAULTARGS
+      args:
+        - TAG
+        - BOINC_USER
+        - PROJECT_ROOT
     volumes:
-     - "project:/root/project.dst"
+      - "project:$PROJECT_ROOT.dst"
+      - "secrets:/run/secrets"
+    hostname: makeproject
     environment:
-     - URL_BASE=$URL_BASE
-     - PROJECT=$PROJECT
+      - URL_BASE
+      - PROJECT
 
   apache:
-    image: boinc/server_apache:latest$TAG
+    image: boinc/server_apache:$VERSION$TAG$DEFAULTARGS
     build: 
       context: images/apache
-      dockerfile: Dockerfile$TAG
+      target: apache$DEFAULTARGS
+      args:
+        - TAG
+        - BOINC_USER
+        - PROJECT_ROOT
     hostname: $PROJECT
-    network_mode: bridge
-    links:
-     - mysql
+    depends_on:
+      - mysql
     volumes: 
-     - "project:/root/project"
-     - "results:/results"
-     - "/var/run/docker.sock:/var/run/docker.sock"
+      - "project:/home/$BOINC_USER/project"
+      - "results:/results"
+      - "secrets:/run/secrets"
+      - "/dev/null:/run/secrets/keys/code_sign_private"
+      - "/var/run/docker.sock:/var/run/docker.sock"
     ports: 
-     - "80:80"
+      - "80:80"
     tty: true
     environment:
-     - URL_BASE=$URL_BASE
-     - PROJECT=$PROJECT
+      - URL_BASE
+      - PROJECT