Jenkins: move "master config" from local NAS to S3 bucket

.gitignore

@@ -1,4 +1,4 @@
 docker/files/id_rsa.pub
 cache
 wincache
-slave.retry
+*.retry

README.md

@@ -89,46 +89,76 @@ Delete unused old AMIs. Delete unused associated "snapshot" volumes.
 First build the nuxeo/jenkins-base image:
 Add your id\_rsa.pub in docker/files/id\_rsa.pub (so ansible can connect later) then:
 
-    cd docker
-    docker build -t nuxeo/jenkins-base .
+    docker build -t nuxeo/jenkins-base docker
 
 You should update that base image now and then to get package updates, that will make the ansible build faster.
 
 
 Run a container from that image, exporting the SSH port locally:
 
-    docker run -d -t -i -p 127.0.0.1:2222:22 --name=slave nuxeo/jenkins-base
+    docker run -d -t -i -p 127.0.0.1:2222:22 --name=slave-common nuxeo/jenkins-base
 
 Make an inventory file for ansible to access this container:
 
     [multidb:children]
     docker
-
     [docker:children]
     slaves
-
     [slaves]
-    container ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1
+    slavecommon ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1
 
 Run ansible normally on this container:
 
-    ansible-playbook -i inventory/slavetmp/hosts slave.yml -v
+    ansible-playbook -i inventory/slave-common/hosts slave-common.yml -v
 
 Commit this container:
 
-    docker commit slave nuxeo/jenkins-docker
+    docker commit slave-common nuxeo/jenkins-common
+
+Run pub/priv containers from that image:
+
+    docker run -d -t -i -p 127.0.0.1:2223:22 --name=slave-pub nuxeo/jenkins-common
+    docker run -d -t -i -p 127.0.0.1:2224:22 --name=slave-priv nuxeo/jenkins-common
+
+Make inventory files to access those containers:
+
+    [multidb:children]
+    docker
+    [docker:children]
+    slaves
+    [slaves]
+    slavepub ansible_ssh_port=2223 ansible_ssh_host=127.0.0.1
+
+    [multidb:children]
+    docker
+    [docker:children]
+    slaves
+    [slaves]
+    slavepriv ansible_ssh_port=2224 ansible_ssh_host=127.0.0.1
+
+Run ansible on those containers:
+
+    ansible-playbook -i inventory/slave-pub/hosts slave-pub.yml -v
+    ansible-playbook -i inventory/slave-priv/hosts slave-priv.yml -v
+
+Commit those containers:
+
+    docker commit slave-pub nuxeo/jenkins-pub
+    docker commit slave-priv nuxeo/jenkins-priv
 
 Tag the image for the remote registry:
 
-    docker tag nuxeo/jenkins-docker dockerpriv.nuxeo.com:443/nuxeo/jenkins-slave
-    docker tag nuxeo/jenkins-docker dockerpriv.nuxeo.com:443/nuxeo/jenkins-ondemand
-    docker tag nuxeo/jenkins-docker dockerpriv.nuxeo.com:443/nuxeo/jenkins-check
+    docker tag nuxeo/jenkins-pub dockerpriv.nuxeo.com:443/nuxeo/jenkins-slave
+    docker tag nuxeo/jenkins-pub dockerpriv.nuxeo.com:443/nuxeo/jenkins-ondemand
+    docker tag nuxeo/jenkins-pub dockerpriv.nuxeo.com:443/nuxeo/jenkins-check
+    docker tag nuxeo/jenkins-priv dockerpriv.nuxeo.com:443/nuxeo/jenkins-slavepriv
 
 Push the image:
 
     docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-slave
     docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-ondemand
     docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-check
+    docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-slavepriv
 
 You can then pull the image on the slaves hosts and restart the slaves containers with the new image.
 

ansible.cfg

@@ -1,3 +1,3 @@
 [ssh_connection]
 #control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r
-control_path = %(directory)s/%%h-%%r
+control_path = %(directory)s/%%h-%%p-%%r

inventory/slave-common/hosts

@@ -0,0 +1,9 @@
+[multidb:children]
+docker
+
+[docker:children]
+slaves
+
+[slaves]
+slavecommon ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1
+

inventory/slavetmp/hosts → inventory/slave-priv/hosts

@@ -5,5 +5,5 @@ docker
 slaves
 
 [slaves]
-container ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1
+slavepriv ansible_ssh_port=2224 ansible_ssh_host=127.0.0.1
 

inventory/slave-pub/hosts

@@ -0,0 +1,9 @@
+[multidb:children]
+docker
+
+[docker:children]
+slaves
+
+[slaves]
+slavepub ansible_ssh_port=2223 ansible_ssh_host=127.0.0.1
+

roles/slave_config_common/tasks/main.yml

@@ -12,6 +12,16 @@
   when: aws_id.msg == "" or aws_secret.msg == ""
 
 # Slave settings
+- name: .ssh dir for ubuntu
+  file: path=/home/ubuntu/.ssh state=directory owner=ubuntu group=ubuntu mode=0700
+- name: Get authorized_keys for ubuntu
+  s3: bucket={{s3_bucket}} region={{s3_region}}
+      object=/{{s3_config_path}}/jenkins_home/.ssh/authorized_keys dest=/home/ubuntu/.ssh/authorized_keys mode=get
+      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different
+- authorized_key: user=ubuntu key="{{item}}"
+  with_file:
+  - Jenkins_aws_system.pem.pub
+
 - name: Get jenkins base home from S3
   command: aws s3 sync s3://{{s3_bucket}}/{{s3_config_path}}/jenkins_home/ /opt/jenkins --region={{s3_region}}
   environment:
@@ -21,6 +31,7 @@
   changed_when: homesync.stdout != ""
 - file: path=/opt/jenkins/.ssh state=directory owner=jenkins group=jenkins mode=700
 - file: path=/opt/jenkins/.ssh/id_rsa state=file owner=jenkins group=jenkins mode=0600
+- file: path=/opt/jenkins/.ssh/authorized_keys state=file owner=jenkins group=jenkins mode=0600
 - file: path=/opt/jenkins/.profile state=file owner=jenkins group=jenkins mode=0600
 - file: path=/opt/jenkins/.netrc state=file owner=jenkins group=jenkins mode=0600
 - file: path=/opt/jenkins/.s3cfg state=file owner=jenkins group=jenkins mode=0600
@@ -29,7 +40,6 @@
   with_file:
   - jenkins_id_rsa.pub
   - Jenkins_aws_system.pem.pub
-- authorized_key: user=jenkins key='{{lookup("pipe", "cat /opt/build/hudson/authorized_keys/id_*.pub" )}}'
 
 - copy: src=91-cloudimg-jenkins dest=/etc/sudoers.d/91-cloudimg-jenkins
         mode=440 owner=root group=root
@@ -72,11 +82,11 @@
 - name: gradle.properties
   s3: bucket={{s3_bucket}} region={{s3_region}}
       object=/{{s3_config_path}}/gradle.properties dest=/opt/build/hudson/gradle.properties mode=get
-      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=false
+      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different
 - name: instance.clid
   s3: bucket={{s3_bucket}} region={{s3_region}}
       object=/{{s3_config_path}}/instance.clid dest=/opt/build/hudson/instance.clid mode=get
-      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=false
+      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different
 - copy: src=use_mnt_ondemand.sh dest=/opt/jenkins/use_mnt.sh
         mode=755 owner=jenkins group=jenkins
   when: "'aws_ondemand' in group_names"
@@ -93,7 +103,7 @@
 - name: Kerberos krb5.conf
   s3: bucket={{s3_bucket}} region={{s3_region}}
       object=/{{s3_config_path}}/keystores/keytabs/krb5.conf dest=/etc/krb5.conf mode=get
-      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=false
+      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different
 
 - name: Setup Azure CLI
   shell: azure account import /opt/build/keystores/azure.nuxeo.com.publishsettings

roles/slave_config_qapriv/tasks/main.yml

@@ -15,7 +15,7 @@
   alternatives: name=firefox link=/usr/bin/firefox path=/opt/build/tools/firefox-24/firefox
 - s3: bucket={{s3_bucket}} region={{s3_region}}
       object=/{{s3_config_path}}/settings.xml dest=//opt/jenkins/.m2/settings.xml mode=get
-      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=true
+      aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different
   become_user: jenkins
 - file: path=/opt/jenkins/.m2/settings.xml state=file owner=jenkins group=jenkins
 

roles/slave_tools/tasks/main.yml

@@ -11,11 +11,6 @@
   fail: msg="You must have AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in your environment"
   when: aws_id.msg == "" or aws_secret.msg == ""
 
-- name: Create user 'ubuntu'
-  authorized_key: user=ubuntu key="{{item}}"
-  with_file:
-  - Jenkins_aws_system.pem.pub
-- authorized_key: user=ubuntu key='{{lookup("pipe", "cat /opt/build/hudson/authorized_keys/id_*.pub" )}}'
 - name: Create user 'jenkins'
   user: name=jenkins home=/opt/jenkins state=present shell=/bin/bash
 - file: path=/opt/jenkins/ state=directory owner=jenkins group=jenkins

slave.yml → slave-common.yml

@@ -12,5 +12,4 @@
   roles:
   - slave_tools
   - slave_config_common
-  - slave_config_qa
 

slave-priv.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  gather_facts: no
+  user: ubuntu
+  become: yes
+  roles:
+  - slave_config_qapriv
+

slave-pub.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  gather_facts: no
+  user: ubuntu
+  become: yes
+  roles:
+  - slave_config_qa
+