placeholder

Makefile

@@ -20,14 +20,11 @@ define deis_units
 endef
 
 # TODO: re-evaluate the fragile start order
-COMPONENTS=builder cache controller database logger registry
-ALL_COMPONENTS=$(COMPONENTS) router
-START_COMPONENTS=registry logger cache database
-
-ALL_UNITS = $(foreach C,$(COMPONENTS),$(wildcard $(C)/systemd/*.service))
+ALL_COMPONENTS=builder cache controller database logger registry router
+START_COMPONENTS=registry logger cache
 START_UNITS = $(foreach C,$(START_COMPONENTS),$(wildcard $(C)/systemd/*.service))
 
-DATA_CONTAINER_TEMPLATES=builder/systemd/deis-builder-data.service database/systemd/deis-database-data.service logger/systemd/deis-logger-data.service registry/systemd/deis-registry-data.service
+DATA_CONTAINER_TEMPLATES=builder/systemd/deis-builder-data.service logger/systemd/deis-logger-data.service registry/systemd/deis-registry-data.service
 
 all: build run
 
@@ -44,8 +41,16 @@ install: check-fleet install-routers
 	$(FLEETCTL) load $(START_UNITS)
 	$(FLEETCTL) load controller/systemd/*.service
 	$(FLEETCTL) load builder/systemd/*.service
+	echo $(shell make install-databases)
 	echo $(shell make install-data-containers)
 
+install-databases: check-fleet
+	@$(foreach U, $(DATABASE_UNITS), \
+		cp database/systemd/deis-database.service ./$(U) ; \
+		$(FLEETCTL) load ./$(U) ; \
+		rm -f ./$(U) ; \
+	)
+
 install-data-containers: check-fleet
 	@$(foreach T, $(DATA_CONTAINER_TEMPLATES), \
 		cp $(T).template . ; \
@@ -77,7 +82,7 @@ rsync:
 
 run: install start
 
-start: check-fleet start-warning start-routers
+start: check-fleet start-warning start-routers start-databases
 	@# registry logger cache database
 	$(call echo_yellow,"Waiting for deis-registry to start...")
 	$(FLEETCTL) start -no-block $(START_UNITS)
@@ -113,6 +118,18 @@ start: check-fleet start-warning start-routers
 
 	$(call echo_yellow,"Your Deis cluster is ready to go! Continue following the README to login and use Deis.")
 
+start-databases: check-fleet
+	$(call echo_yellow,"Waiting for 1 of $(DEIS_NUM_DATABASES) deis-databases to start...")
+	$(foreach U,$(DATABASE_UNITS),$(FLEETCTL) start -no-block $(U);)
+	@until $(FLEETCTL) list-units | egrep -q "deis-database.+(running)"; \
+		do sleep 2; \
+			printf "\033[0;33mStatus:\033[0m "; $(FLEETCTL) list-units | \
+			grep "deis-database" | head -n 1 | \
+			awk '{printf "%-10s (%s)    \r", $$4, $$5}'; \
+			sleep 8; \
+		done
+	$(call check_for_errors)
+
 start-routers: check-fleet start-warning
 	$(call echo_yellow,"Waiting for 1 of $(DEIS_NUM_ROUTERS) deis-routers to start...")
 	$(foreach R,$(ROUTER_UNITS),$(FLEETCTL) start -no-block $(R);)

controller/templates/confd_settings.py

@@ -13,8 +13,8 @@
         'NAME': '{{ .deis_database_name }}',
         'USER': '{{ .deis_database_user }}',
         'PASSWORD': '{{ .deis_database_password }}',
-        'HOST': '{{ .deis_database_host }}',
-        'PORT': '{{ .deis_database_port }}',
+        'HOST': '{{ .deis_database_master_host }}',
+        'PORT': '{{ .deis_database_master_port }}',
     }
 }
 

database/Dockerfile

@@ -8,9 +8,6 @@ RUN wget --quiet --no-check-certificate -O - https://www.postgresql.org/media/ke
 RUN apt-get update
 RUN apt-get install -yq postgresql-9.3 && /etc/init.d/postgresql stop
 
-# debug to remove
-RUN apt-get install -yq curl
-
 # add the current build context to /app
 ADD . /app
 

database/Makefile

@@ -5,17 +5,33 @@ build:
 	$(call ssh_all,'cd share/database && sudo docker build -t deis/database .')
 
 install: check-fleet
-	$(FLEETCTL) load systemd/*
+	@$(foreach U, $(DATABASE_UNITS), \
+		cp systemd/deis-database.service ./$(U) ; \
+		$(FLEETCTL) load ./$(U) ; \
+		rm -f ./$(U) ; \
+	)
 
 uninstall: check-fleet stop
-	$(FLEETCTL) unload systemd/*
-	$(FLEETCTL) destroy systemd/*
+	@$(foreach U, $(DATABASE_UNITS), \
+		cp systemd/deis-database.service ./$(U) ; \
+		$(FLEETCTL) unload ./$(U) ; \
+		$(FLEETCTL) destroy ./$(U) ; \
+		rm -f ./$(U) ; \
+	)
 
 start: check-fleet
-	$(FLEETCTL) start -no-block systemd/*
+	@$(foreach U, $(DATABASE_UNITS), \
+		cp systemd/deis-database.service ./$(U) ; \
+		$(FLEETCTL) start -no-block ./$(U) ; \
+		rm -f ./$(U) ; \
+	)
 
 stop: check-fleet
-	$(FLEETCTL) stop -block-attempts=600 systemd/*
+	@$(foreach U, $(DATABASE_UNITS), \
+		cp systemd/deis-database.service ./$(U) ; \
+		$(FLEETCTL) stop -block-attempts=600 ./$(U) ; \
+		rm -f ./$(U) ; \
+	)
 
 restart: stop start
 

database/bin/attempt_master

@@ -0,0 +1,12 @@
+#!/bin/bash
+# attempt to become the Postgres master
+
+# First, is there currently a master?
+if etcdctl --no-sync -C $ETCD mk /deis/database/master $HOST --ttl $ETCD_TTL >/dev/null; then
+  exit 0
+# Are we the master?
+elif etcdctl --no-sync -C $ETCD set /deis/database/master $HOST --ttl $ETCD_TTL --swap-with-value "$HOST" >/dev/null; then
+  exit 0
+fi
+
+exit 1

database/bin/boot

@@ -24,19 +24,24 @@ fi
 
 # wait for etcd to be available
 until etcdctl --no-sync -C $ETCD ls >/dev/null; do
-	echo "waiting for etcd at $ETCD..."
-	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
+  echo "waiting for etcd at $ETCD..."
+  sleep $(($ETCD_TTL/2))  # sleep for half the TTL
 done
 
 # wait until etcd has discarded potentially stale values
 sleep $(($ETCD_TTL+1))
 
+function etcd_safe_mkdir {
+  etcdctl --no-sync -C $ETCD mkdir $1 >/dev/null 2>&1 || true
+}
+
 function etcd_safe_set {
   if ! etcdctl --no-sync -C $ETCD get $ETCD_PATH/$1 >/dev/null 2>&1; then
     etcdctl --no-sync -C $ETCD set $ETCD_PATH/$1 $2 >/dev/null
   fi
 }
 
+etcd_safe_mkdir /deis/database
 etcd_safe_set engine postgresql_psycopg2
 etcd_safe_set adminUser ${PG_ADMIN_USER:-postgres}
 etcd_safe_set adminPass ${PG_ADMIN_PASS:-changeme123}
@@ -46,8 +51,8 @@ etcd_safe_set name ${PG_USER_DB:-deis}
 
 # wait for confd to run once and install initial templates
 until confd -onetime -node $ETCD -config-file /app/confd.toml; do
-	echo "database: waiting for confd to write initial templates..."
-	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
+  echo "database: waiting for confd to write initial templates..."
+  sleep $(($ETCD_TTL/2))  # sleep for half the TTL
 done
 
 # run the service in the background
@@ -80,24 +85,33 @@ echo deis-database running...
 # publish the service to etcd using the injected HOST and PORT
 if [[ ! -z $PUBLISH ]]; then
 
-	# configure service discovery
-	PORT=${PORT:-5432}
-	PROTO=${PROTO:-tcp}
+  # configure service discovery
+  PORT=${PORT:-5432}
+  PROTO=${PROTO:-tcp}
+
+  set +e
+
+  # wait for the service to become available on PUBLISH port
+  sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
 
-	set +e
+  # while the port is listening, publish to etcd
+  while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
+    etcdctl --no-sync -C $ETCD set /deis/database/$HOST/host $HOST --ttl $ETCD_TTL >/dev/null
+    etcdctl --no-sync -C $ETCD set /deis/database/$HOST/port $PORT --ttl $ETCD_TTL >/dev/null
 
-	# wait for the service to become available on PUBLISH port
-	sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
+    # now, we try to become master
+    if /app/bin/attempt_master; then
+      /app/bin/promote_to_master
+    else
+      /app/bin/demote_to_slave
+    fi
 
-	# while the port is listening, publish to etcd
-	while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
-		etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
-		etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PORT --ttl $ETCD_TTL >/dev/null
-		sleep $(($ETCD_TTL/2)) # sleep for half the TTL
-	done
+    # try it all again soon
+    sleep $(($ETCD_TTL/2)) # sleep for half the TTL
+  done
 
-	# if the loop quits, something went wrong
-	exit 1
+  # if the loop quits, something went wrong
+  exit 1
 
 fi
 

database/bin/demote_to_slave

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# return success if we are already a slave
+
+exit 0

database/bin/promote_to_master

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# return success if we are already master
+
+exit 0

database/systemd/deis-database.service

@@ -1,18 +1,16 @@
 [Unit]
 Description=deis-database
-Requires=deis-database-data.service
-After=deis-database-data.service
 
 [Service]
 EnvironmentFile=/etc/environment
 TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/database`; docker history $IMAGE >/dev/null || docker pull $IMAGE"
 ExecStartPre=/bin/sh -c "docker inspect deis-database >/dev/null && docker rm -f deis-database || true"
-ExecStart=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/database` && docker run --name deis-database --rm -p 5432:5432 -e PUBLISH=5432 -e HOST=$COREOS_PRIVATE_IPV4 --volumes-from deis-database-data $IMAGE"
+ExecStart=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/database` && docker run --name deis-database -p 5432:5432 -e PUBLISH=5432 -e HOST=$COREOS_PRIVATE_IPV4 $IMAGE"
 ExecStop=/bin/bash -c "nsenter --pid --uts --mount --ipc --net --target $(docker inspect --format='{{ .State.Pid }}' deis-database) sudo service postgresql stop"
 
 [Install]
 WantedBy=multi-user.target
 
 [X-Fleet]
-X-ConditionMachineOf=deis-database-data.service
+X-Conflicts=deis-database.*.service

docs/installing_deis/upgrading-deis.rst

@@ -120,15 +120,13 @@ You can ``make status`` to ensure that all services are stopped.
 
 Export data containers
 ^^^^^^^^^^^^^^^^^^^^^^
-Four Deis components, builder, database, logger, and registry, run separate containers to store
+Three Deis components, builder, logger, and registry, run separate containers to store
 their stateful data. We export these as tarballs before upgrading the containers.
 
 .. code-block:: console
 
     dev $ fleetctl ssh deis-builder.service
     coreos $ sudo docker export deis-builder-data > /home/coreos/deis-builder-data-backup.tar
-    dev $ fleetctl ssh deis-database.service
-    coreos $ sudo docker export deis-database-data > /home/coreos/deis-database-data-backup.tar
     dev $ fleetctl ssh deis-logger.service
     coreos $ sudo docker export deis-logger-data > /home/coreos/deis-logger-data-backup.tar
     dev $ fleetctl ssh deis-registry.service
@@ -158,8 +156,6 @@ We need to reimport the saved data containers we exported earlier.
 
     dev $ fleetctl ssh deis-builder.service
     coreos $ cat /home/coreos/deis-builder-data-backup.tar | sudo docker import - deis-builder-data
-    dev $ fleetctl ssh deis-database.service
-    coreos $ cat /home/coreos/deis-database-data-backup.tar | sudo docker import - deis-database-data
     dev $ fleetctl ssh deis-logger.service
     coreos $ cat /home/coreos/deis-logger-data-backup.tar | sudo docker import - deis-logger-data
     dev $ fleetctl ssh deis-registry.service

docs/managing_deis/backing_up_data.rst

@@ -12,15 +12,13 @@ Deis maintains platform state in two places: data containers and etcd.
 Data containers
 ---------------
 Data containers are simply Docker containers that expose a volume which is shared with another container.
-The components with data containers are builder, database, logger, and registry. Since these are just
+The components with data containers are builder, logger, and registry. Since these are just
 Docker containers, they can be exported with ordinary Docker commands:
 
 .. code-block:: console
 
     dev $ fleetctl ssh deis-builder.service
     coreos $ sudo docker export deis-builder-data > /home/coreos/deis-builder-data-backup.tar
-    dev $ fleetctl ssh deis-database.service
-    coreos $ sudo docker export deis-database-data > /home/coreos/deis-database-data-backup.tar
     dev $ fleetctl ssh deis-logger.service
     coreos $ sudo docker export deis-logger-data > /home/coreos/deis-logger-data-backup.tar
     dev $ fleetctl ssh deis-registry.service
@@ -32,8 +30,6 @@ Importing looks very similar:
 
     dev $ fleetctl ssh deis-builder.service
     coreos $ cat /home/coreos/deis-builder-data-backup.tar | sudo docker import - deis-builder-data
-    dev $ fleetctl ssh deis-database.service
-    coreos $ cat /home/coreos/deis-database-data-backup.tar | sudo docker import - deis-database-data
     dev $ fleetctl ssh deis-logger.service
     coreos $ cat /home/coreos/deis-logger-data-backup.tar | sudo docker import - deis-logger-data
     dev $ fleetctl ssh deis-registry.service

docs/managing_deis/controller_settings.rst

@@ -35,23 +35,22 @@ Settings used by controller
 ---------------------------
 The following etcd keys are used by the controller component.
 
-====================================      ======================================================
+====================================      =================================================================
 setting                                   description
-====================================      ======================================================
+====================================      =================================================================
 /deis/controller/registrationEnabled      enable registration for new Deis users (default: true)
 /deis/controller/webEnabled               enable controller web UI (default: false)
 /deis/cache/host                          host of the cache component (set by cache)
 /deis/cache/port                          port of the cache component (set by cache)
-/deis/database/host                       host of the database component (set by database)
-/deis/database/port                       port of the database component (set by database)
+/deis/database/master                     host:port of the database component acting as master (set by database)
 /deis/database/engine                     database engine (set by database)
 /deis/database/name                       database name (set by database)
 /deis/database/user                       database user (set by database)
 /deis/database/password                   database password (set by database)
 /deis/registry/host                       host of the registry component (set by registry)
 /deis/registry/port                       port of the registry component (set by registry)
 /deis/registry/protocol                   protocol of the registry component (set by registry)
-====================================      ======================================================
+====================================      =================================================================
 
 Using a custom controller image
 -------------------------------

docs/managing_deis/database_settings.rst

@@ -19,18 +19,19 @@ Settings set by database
 ------------------------
 The following etcd keys are set by the database component, typically in its /bin/boot script.
 
-===========================              =================================================================================
+===========================              ====================================================================================
 setting                                  description
-===========================              =================================================================================
-/deis/database/host                      IP address of the host running database
-/deis/database/port                      port used by the database service (default: 5432)
+===========================              ====================================================================================
+/deis/database/$HOST/host                IP address of the host running this database (there can be multiple databases)
+/deis/database/$HOST/port                port used by this database service (there can be multiple databases) (default: 5432)
+/deis/database/master                    host:port of the database instance currently acting as master
 /deis/database/engine                    database engine (default: postgresql_psycopg2)
 /deis/database/adminUser                 database admin user (default: postgres)
 /deis/database/adminPass                 database admin password (default: changeme123)
 /deis/database/user                      database user (default: deis)
 /deis/database/password                  database password (default: changeme123)
 /deis/database/name                      database name (default: deis)
-===========================              =================================================================================
+===========================              ====================================================================================
 
 Settings used by database
 -------------------------