zalando · lotharschulz · Nov 21, 2017 · Sep 8, 2017 · Sep 15, 2017 · Sep 15, 2017
diff --git a/DockerfileAutomata → Dockerfile b/DockerfileAutomata → Dockerfile
@@ -1,23 +1,36 @@
-FROM registry.opensource.zalan.do/stups/python:3.5-cd26
+FROM registry.opensource.zalan.do/stups/python:3.6.2-14
 MAINTAINER lothar.schulz@zalando.de
 
-USER root
 # folder structure and user
 RUN \
-  useradd -d /backup -u 998 -o application && \
-  mkdir -p /data/ghe-production-data/ && mkdir -p /backup/backup-utils/ && \
+# read package lists
+  apt-get update -y && \
+  apt-get install -y sudo && \
+# create application user
+  useradd -d /backup -u 998 -o -c "application user" application && \
+# allow su
+  echo "application ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/application && \
+  chmod 0440 /etc/sudoers.d/application && \
+# update w/ latest security patches
+# install python pip3 pyyaml & english, git, screen
+  apt-get install -y --no-install-recommends unattended-upgrades python3=3.5.1-3 python3-dev=3.5.1-3 && \
+  apt-get install -y --no-install-recommends python3-pip=8.1.1-2ubuntu0.4 python3-yaml=3.11-3build1 && \
+  apt-get install -y --no-install-recommends language-pack-en=1:16.04+20161009 git=1:2.7.4-0ubuntu1.3 && \
+  apt-get install -y --no-install-recommends ssh=1:7.2p2-4ubuntu2.2 && \
+  apt-get install -y --no-install-recommends bash=4.3-14ubuntu1.2 && \
+  apt-get install -y --no-install-recommends rsync=3.1.1-3ubuntu1 && \
+  apt-get install -y --no-install-recommends cron=3.0pl1-128ubuntu2 && \
+# install boto3
+  pip3 install --upgrade boto==2.48.0 boto3==1.4.7 && \
+# clean apt-get lists
+  rm -rf /var/lib/apt/lists/* && \
+# create directories
+  mkdir -p /backup/backup-utils/ && \
   mkdir -p /kms && mkdir -p /var/log/ && mkdir /delete-instuck-backups
 WORKDIR /backup
 
-# read package lists
-# update w/ latest security patches
-# install python pip3 boto3 pyyaml & english & git
-# clone backup-utils
 RUN \
-  apt-get update -y && \
-  apt-get install -y unattended-upgrades python3 python3-dev python3-pip python3-yaml language-pack-en git && \
-  pip3 install --upgrade boto boto3 && \
-  rm -rf /var/lib/apt/lists/* && \
+# clone backup-utils
   git clone -b stable https://github.com/github/backup-utils.git && \
   git -C /backup/backup-utils pull
 
@@ -33,31 +46,33 @@ COPY start_backup.sh /start_backup.sh
 COPY python/delete_instuck_progress.py /delete-instuck-backups/delete_instuck_progress.py
 
 # copy cron job
-COPY cron-ghe-backup-automata /etc/cron.d/ghe-backup
+COPY cron-ghe-backup /etc/cron.d/ghe-backup
 
 # copy finale CMD commands
 COPY final-docker-cmd.sh /backup/final-docker-cmd.sh
 
+
 #PLACEHOLDER_4_COPY_SCM_SOURCE_JSON
 
-# change mode of files
 RUN \
-  chown -R application: /data && \
+# change mode of files
   chown -R application: /backup && \
   chown -R application: /kms && \
   chown -R application: /delete-instuck-backups && \
-  chown -R root: /start_backup.sh && \
+  chown -R application: /start_backup.sh && \
   chmod 0700 /kms/extract_decrypt_kms.py && \
   chmod 0700 /kms/convert-kms-private-ssh-key.sh && \
   chmod 0644 /etc/cron.d/ghe-backup && \
   chmod 0700 /delete-instuck-backups/delete_instuck_progress.py && \
   chmod 0700 /start_backup.sh && \
   chmod 0700 /backup/final-docker-cmd.sh && \
-  mkfifo /var/log/ghe-prod-backup.log
-
-# delete_instuck_progress log
-RUN \
+  mkfifo /var/log/ghe-prod-backup.log && \
+  chown -R application: /var/log/ghe-prod-backup.log && \
   touch /var/log/ghe-delete-instuck-progress.log && \
   chown -R application: /var/log/ghe-delete-instuck-progress.log
 
-CMD ["/backup/final-docker-cmd.sh"]
+USER application
+
+# https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#user mentions to avoid sudo,
+# however cron as part of the final-docker-cmd.sh has to run as
+CMD "/backup/final-docker-cmd.sh"
diff --git a/DockerfileBus b/DockerfileBus
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -47,14 +47,14 @@ node('kraken') {
 
 node('kraken') {
     stage("Build Docker Image") {
-        docker(dockerRepo, fullImageName, "DockerfileAutomata" , false)
+        docker(dockerRepo, fullImageName, "Dockerfile" , false)
     }
 }
 
 if (env.BRANCH_NAME == 'master') {
     node('kraken') {
         stage("Build and Push Docker") {
-            docker(dockerRepo, fullImageName, "DockerfileAutomata" , true)
+            docker(dockerRepo, fullImageName, "Dockerfile" , true)
         }
     }
     timeout(time: 60, unit: "MINUTES") {
@@ -99,8 +99,7 @@ if (env.BRANCH_NAME == 'master') {
 def docker(String dockerRepo, String fullImageName, String dockerfile, boolean pushImage) {
     if (pushImage == true) {
         sh "/tools/run :stups -- scm-source"
-        sh "/tools/run :stups -- sed -i 's&.*#PLACEHOLDER_4_COPY_SCM_SOURCE_JSON.*&COPY scm-source.json /scm-source.json&' DockerfileBus"
-        sh "/tools/run :stups -- sed -i 's&.*#PLACEHOLDER_4_COPY_SCM_SOURCE_JSON.*&COPY scm-source.json /scm-source.json&' DockerfileAutomata"
+        sh "/tools/run :stups -- sed -i 's&.*#PLACEHOLDER_4_COPY_SCM_SOURCE_JSON.*&COPY scm-source.json /scm-source.json&' Dockerfile"
     }
 
     sh "/tools/run :stups -- docker build --rm -t $fullImageName -f $dockerfile ."

diff --git a/README.md b/README.md
@@ -7,11 +7,32 @@
 [Zalando Tech's ](https://tech.zalando.com/) [Github Enterprise](https://enterprise.github.com/) backup approach.
 
 ## Overview
-[Github Enterprise](https://enterprise.github.com/) at Zalando Tech is a Ha setup running master and replica instances on AWS. The AWS account that runs the [high availability](https://help.github.com/enterprise/2.5/admin/guides/installation/high-availability-configuration/) setup also runs one backup host. There is a second backup host running in a different AWS account. We believe this backup gives us reliable backup data even in case one AWS is compromised.
+[Github Enterprise](https://enterprise.github.com/) at Zalando Tech is a 
+[high availability](https://help.github.com/enterprise/2.11/admin/guides/installation/configuring-github-enterprise-for-high-availability/) 
+setup running master and replica instances on AWS. 
+The AWS account that runs the [high availability](https://help.github.com/enterprise/2.11/admin/guides/installation/configuring-github-enterprise-for-high-availability/) 
+setup also runs one backup host. 
+Another backup host can run in a different AWS account. 
+[Zalando Tech's ](https://tech.zalando.com/) [Github Enterprise](https://enterprise.github.com/) backup
+can also run as a [POD](https://kubernetes.io/docs/concepts/workloads/pods/pod/#what-is-a-pod) 
+inside a [Kubernetes](https://kubernetes.io/) cluster.
+
+We believe this backup approach provides reliable backup data even in case one AWS account or Kubernetes cluster is compromised.
+
+![overview](/ZalandoGithubEnterprise.jpg "backup approach overview")
+
+Basically [Zalando Tech's ](https://tech.zalando.com/) [Github Enterprise](https://enterprise.github.com/) backup
+wraps github's [backup-utils](https://github.com/github/backup-utils) in a 
+[Docker](https://www.docker.com/) container. 
+If running on AWS, an [EBS volume](https://aws.amazon.com/de/ebs/) stores the actual backup data.
+This way one can access the data even if the regarding backup host is down.
+If running on Kubernetes, a [stateful set](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/) 
+including [volumes](https://kubernetes.io/docs/concepts/storage/volumes/) and 
+[volume claims](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) stores the actual backup data.
+See a sample [statefulset below]()https://github.com/zalando/ghe-backup/blob/master/README.md#kubernetes-stateful-set,-volume,-volume-claim)
+[Zalando Kubernetes](https://github.com/zalando-incubator/kubernetes-on-aws#kubernetes-on-aws) is based on AWS, so [volume claims
+ are based on EBS](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#aws).
 
-![overview](/backup_overview.PNG "backup approach overview")
-
-Basically ghe-backup wraps github's [backup-utils](https://github.com/github/backup-utils) in a [Docker](https://www.docker.com/) container. An [EBS volume](https://aws.amazon.com/de/ebs/) stores the actual backup data to be able to access the data even if the regarding backup host is down.
 
 ## Local docker development
 
@@ -43,8 +64,13 @@ e.g.
 
 ### IAM [policy](http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies.html) settings
 
-Github Enterprise backup hosts contain private ssh keys that have to match with public ssh keys registered on the Github Enterprise main instance.
-Private ssh keys should not be propagated unencrypted with deployments. AWS KMS allows to encrypt any kind of data, so this service is used to encrypt the private ssh key. KMS actions are managed by policies to make sure only configured tasks can be performed.
+[Zalando Tech's ](https://tech.zalando.com/) [Github Enterprise](https://enterprise.github.com/) backup hosts contain 
+private ssh keys that have to match with public ssh keys registered on the Github Enterprise main instance.
+Private ssh keys should not be propagated unencrypted with deployments. 
+AWS KMS allows to encrypt any kind of data, so this service is used to encrypt the private ssh key for both, 
+[Zalando Tech's ](https://tech.zalando.com/) [Github Enterprise](https://enterprise.github.com/) backup running on AWS and Kubernetes. 
+KMS actions are managed by policies to make sure only configured tasks can be performed.
+
 A kms policy similar to the one shown below is needed to:   
 * allow kms decryption of the encrypted ssh key
 * access s3 bucket
@@ -109,6 +135,31 @@ Pls go to bashtest directory:
 
 *Make sure* you run ```./cleanup-tests.sh ``` in order to clean up afterwards.  
 
+
+### Running in an additional AWS account
+Please adapt the cron tab definitions when running in another AWS account e.g. to the values in cron-ghe-backup-alternative.
+This lowers the load on the Github Enterprise master with respect to backup attempts.
+
+
+### Restore
+
+Restoring backups is based on github's _(using the backup and restore commands)[https://github.com/github/backup-utils#using-the-backup-and-restore-commands]_.
+The actual _ghe-restore_ command gets issued from the backup host. Please note: the backup restore can run for several hours.
+(Nohup)[https://en.wikipedia.org/wiki/Nohup] is recommended to keep the restore process running even if the shell connection is lost.
+
+sample steps include:
+```
+put ghe instance to restor to into maintenance mode
+# ssh into your ec2 instance and exec into your container
+# docker exec -it [container label or ID] bash/sh
+# or 
+# exec into your pod
+# kubectl exec -it [your pod e.g. statefulset-ghe-backup-0] bash/sh
+nohup /backup/backup-utils/bin/ghe-restore -f [IP address of the ghe master to restore] &
+# monitor the backup progress
+tail -f nohup.out
+```
+
 ## Contribution
 pls refer to [CONTRIBUTING.md](CONTRIBUTING.md)
 
@@ -179,10 +230,65 @@ and run it like:
 ### EBS volumes with Senza
 Please follow these instructions: [senza's storage guild](https://docs.stups.io/en/latest/user-guide/storage.html) to create a EBS volume the stups way.
 
-## Blog Post
-You can find more context and details on [Zalando's](https://github.com/zalando/) [tech blog](https://tech.zalando.com/).  
-Blog post one is basically why and how ghe-backup was done: [tech.zalando.com/blog/multi-aws-github-enterprise-backup](https://tech.zalando.com/blog/multi-aws-github-enterprise-backup/).  
-Blog post two explains how changes gets continuously delivered using [Lizzy](https://github.com/zalando/lizzy/): [tech.zalando.com/blog/ci-pipelines-with-lizzy](https://tech.zalando.com/blog/ci-pipelines-with-lizzy/).  
+### Kubernetes stateful set, volume, volume claim
+
+The statefulset resource definition is the main kubernetes configuration file:
+```
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+      name: statefulset-ghe-backup
+spec:
+  serviceName: deploy-ghe-backup
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: ghe-backup
+      annotations:
+        pod.alpha.kubernetes.io/initialized: "true"
+    spec:
+      containers:
+      - name: container-{ghe-backup}
+        image: pierone.zalando/machinery/ghe-backup-kubernetes:latest
+        resources:
+          requests:
+            cpu: 100m
+            memory: 1Gi
+          limits:
+            cpu: 400m
+            memory: 4Gi
+        volumeMounts:
+        - name: data-{ghe-backup}
+          mountPath: /data
+        - name: {ghe-backup}-secret
+          mountPath: /meta/ghe-backup-secret
+          readOnly: true
+        - name: podinfo
+          mountPath: /details
+          readOnly: false
+      volumes:
+      - name: {ghe-backup}-secret
+        secret:
+          secretName: {ghe-backup}-secret
+      - name: podinfo
+        downwardAPI:
+          items:
+            - path: "labels"
+              fieldRef:
+                fieldPath: metadata.labels
+  volumeClaimTemplates:
+  - metadata:
+      name: data-ghe-backup
+      annotations:
+        volume.beta.kubernetes.io/storage-class: standard
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1000Gi 
+```
 
 ===
 ### License

diff --git a/ZalandoGithubEnterprise.jpg b/ZalandoGithubEnterprise.jpg
diff --git a/backup.config b/backup.config
@@ -1,4 +1,4 @@
-GHE_HOSTNAME="github.bus.zalan.do"
+GHE_HOSTNAME="github-master.bus.zalan.do"
 GHE_DATA_DIR="/data/ghe-production-data"
 GHE_NUM_SNAPSHOTS=40
 GHE_EXTRA_SSH_OPTS=" -o StrictHostKeyChecking=no "

diff --git a/backup_overview.PNG b/backup_overview.PNG