diff --git a/.gitignore b/.gitignore index babf066..7835152 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ docker/files/id_rsa.pub cache wincache -slave.retry +*.retry diff --git a/README.md b/README.md index b57f55c..2ba79cf 100644 --- a/README.md +++ b/README.md @@ -89,46 +89,76 @@ Delete unused old AMIs. Delete unused associated "snapshot" volumes. First build the nuxeo/jenkins-base image: Add your id\_rsa.pub in docker/files/id\_rsa.pub (so ansible can connect later) then: - cd docker - docker build -t nuxeo/jenkins-base . + docker build -t nuxeo/jenkins-base docker You should update that base image now and then to get package updates, that will make the ansible build faster. Run a container from that image, exporting the SSH port locally: - docker run -d -t -i -p 127.0.0.1:2222:22 --name=slave nuxeo/jenkins-base + docker run -d -t -i -p 127.0.0.1:2222:22 --name=slave-common nuxeo/jenkins-base Make an inventory file for ansible to access this container: [multidb:children] docker - [docker:children] slaves - [slaves] - container ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1 + slavecommon ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1 Run ansible normally on this container: - ansible-playbook -i inventory/slavetmp/hosts slave.yml -v + ansible-playbook -i inventory/slave-common/hosts slave-common.yml -v Commit this container: - docker commit slave nuxeo/jenkins-docker + docker commit slave-common nuxeo/jenkins-common + +Run pub/priv containers from that image: + + docker run -d -t -i -p 127.0.0.1:2223:22 --name=slave-pub nuxeo/jenkins-common + docker run -d -t -i -p 127.0.0.1:2224:22 --name=slave-priv nuxeo/jenkins-common + +Make inventory files to access those containers: + + [multidb:children] + docker + [docker:children] + slaves + [slaves] + slavepub ansible_ssh_port=2223 ansible_ssh_host=127.0.0.1 + + [multidb:children] + docker + [docker:children] + slaves + [slaves] + slavepriv ansible_ssh_port=2224 ansible_ssh_host=127.0.0.1 + +Run ansible on those containers: + + ansible-playbook -i inventory/slave-pub/hosts slave-pub.yml -v + ansible-playbook -i inventory/slave-priv/hosts slave-priv.yml -v + +Commit those containers: + + docker commit slave-pub nuxeo/jenkins-pub + docker commit slave-priv nuxeo/jenkins-priv Tag the image for the remote registry: - docker tag nuxeo/jenkins-docker dockerpriv.nuxeo.com:443/nuxeo/jenkins-slave - docker tag nuxeo/jenkins-docker dockerpriv.nuxeo.com:443/nuxeo/jenkins-ondemand - docker tag nuxeo/jenkins-docker dockerpriv.nuxeo.com:443/nuxeo/jenkins-check + docker tag nuxeo/jenkins-pub dockerpriv.nuxeo.com:443/nuxeo/jenkins-slave + docker tag nuxeo/jenkins-pub dockerpriv.nuxeo.com:443/nuxeo/jenkins-ondemand + docker tag nuxeo/jenkins-pub dockerpriv.nuxeo.com:443/nuxeo/jenkins-check + docker tag nuxeo/jenkins-priv dockerpriv.nuxeo.com:443/nuxeo/jenkins-slavepriv Push the image: docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-slave docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-ondemand docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-check + docker push dockerpriv.nuxeo.com:443/nuxeo/jenkins-slavepriv You can then pull the image on the slaves hosts and restart the slaves containers with the new image. diff --git a/ansible.cfg b/ansible.cfg index d8cde63..b74d6b4 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,3 +1,3 @@ [ssh_connection] #control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r -control_path = %(directory)s/%%h-%%r +control_path = %(directory)s/%%h-%%p-%%r diff --git a/inventory/slave-common/hosts b/inventory/slave-common/hosts new file mode 100644 index 0000000..6c4b557 --- /dev/null +++ b/inventory/slave-common/hosts @@ -0,0 +1,9 @@ +[multidb:children] +docker + +[docker:children] +slaves + +[slaves] +slavecommon ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1 + diff --git a/inventory/slavetmp/hosts b/inventory/slave-priv/hosts similarity index 51% rename from inventory/slavetmp/hosts rename to inventory/slave-priv/hosts index c21a6b1..7407f29 100644 --- a/inventory/slavetmp/hosts +++ b/inventory/slave-priv/hosts @@ -5,5 +5,5 @@ docker slaves [slaves] -container ansible_ssh_port=2222 ansible_ssh_host=127.0.0.1 +slavepriv ansible_ssh_port=2224 ansible_ssh_host=127.0.0.1 diff --git a/inventory/slave-pub/hosts b/inventory/slave-pub/hosts new file mode 100644 index 0000000..5e7edae --- /dev/null +++ b/inventory/slave-pub/hosts @@ -0,0 +1,9 @@ +[multidb:children] +docker + +[docker:children] +slaves + +[slaves] +slavepub ansible_ssh_port=2223 ansible_ssh_host=127.0.0.1 + diff --git a/roles/slave_config_common/tasks/main.yml b/roles/slave_config_common/tasks/main.yml index 2dcb14e..dcb7497 100644 --- a/roles/slave_config_common/tasks/main.yml +++ b/roles/slave_config_common/tasks/main.yml @@ -12,6 +12,16 @@ when: aws_id.msg == "" or aws_secret.msg == "" # Slave settings +- name: .ssh dir for ubuntu + file: path=/home/ubuntu/.ssh state=directory owner=ubuntu group=ubuntu mode=0700 +- name: Get authorized_keys for ubuntu + s3: bucket={{s3_bucket}} region={{s3_region}} + object=/{{s3_config_path}}/jenkins_home/.ssh/authorized_keys dest=/home/ubuntu/.ssh/authorized_keys mode=get + aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different +- authorized_key: user=ubuntu key="{{item}}" + with_file: + - Jenkins_aws_system.pem.pub + - name: Get jenkins base home from S3 command: aws s3 sync s3://{{s3_bucket}}/{{s3_config_path}}/jenkins_home/ /opt/jenkins --region={{s3_region}} environment: @@ -21,6 +31,7 @@ changed_when: homesync.stdout != "" - file: path=/opt/jenkins/.ssh state=directory owner=jenkins group=jenkins mode=700 - file: path=/opt/jenkins/.ssh/id_rsa state=file owner=jenkins group=jenkins mode=0600 +- file: path=/opt/jenkins/.ssh/authorized_keys state=file owner=jenkins group=jenkins mode=0600 - file: path=/opt/jenkins/.profile state=file owner=jenkins group=jenkins mode=0600 - file: path=/opt/jenkins/.netrc state=file owner=jenkins group=jenkins mode=0600 - file: path=/opt/jenkins/.s3cfg state=file owner=jenkins group=jenkins mode=0600 @@ -29,7 +40,6 @@ with_file: - jenkins_id_rsa.pub - Jenkins_aws_system.pem.pub -- authorized_key: user=jenkins key='{{lookup("pipe", "cat /opt/build/hudson/authorized_keys/id_*.pub" )}}' - copy: src=91-cloudimg-jenkins dest=/etc/sudoers.d/91-cloudimg-jenkins mode=440 owner=root group=root @@ -72,11 +82,11 @@ - name: gradle.properties s3: bucket={{s3_bucket}} region={{s3_region}} object=/{{s3_config_path}}/gradle.properties dest=/opt/build/hudson/gradle.properties mode=get - aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=false + aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different - name: instance.clid s3: bucket={{s3_bucket}} region={{s3_region}} object=/{{s3_config_path}}/instance.clid dest=/opt/build/hudson/instance.clid mode=get - aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=false + aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different - copy: src=use_mnt_ondemand.sh dest=/opt/jenkins/use_mnt.sh mode=755 owner=jenkins group=jenkins when: "'aws_ondemand' in group_names" @@ -93,7 +103,7 @@ - name: Kerberos krb5.conf s3: bucket={{s3_bucket}} region={{s3_region}} object=/{{s3_config_path}}/keystores/keytabs/krb5.conf dest=/etc/krb5.conf mode=get - aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=false + aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different - name: Setup Azure CLI shell: azure account import /opt/build/keystores/azure.nuxeo.com.publishsettings diff --git a/roles/slave_config_qapriv/tasks/main.yml b/roles/slave_config_qapriv/tasks/main.yml index 74c59c1..2e8c854 100644 --- a/roles/slave_config_qapriv/tasks/main.yml +++ b/roles/slave_config_qapriv/tasks/main.yml @@ -15,7 +15,7 @@ alternatives: name=firefox link=/usr/bin/firefox path=/opt/build/tools/firefox-24/firefox - s3: bucket={{s3_bucket}} region={{s3_region}} object=/{{s3_config_path}}/settings.xml dest=//opt/jenkins/.m2/settings.xml mode=get - aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=true + aws_access_key={{aws_id.msg}} aws_secret_key={{aws_secret.msg}} overwrite=different become_user: jenkins - file: path=/opt/jenkins/.m2/settings.xml state=file owner=jenkins group=jenkins diff --git a/roles/slave_tools/files/Jenkins_aws_system.pem.pub b/roles/slave_tools/files/Jenkins_aws_system.pem.pub deleted file mode 100644 index 1f2e526..0000000 --- a/roles/slave_tools/files/Jenkins_aws_system.pem.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCh8PcvyoH/hA4zCTPqodWo/ipGO+/P8V1qbXEqN60BYro5/eG8O5ghZJpprLBnU6AQhiecNiEJetKmVOCT1LX/ALrOSJ6VRkYxrloQw06FixNPLdbZYsFBOrQDJGT59/qR0E1pqxlauU7vMDdjvGgP0AEyc5Hn7woVMOd22VxcWoC6AW4aasRUfrFPSlsrYKg6NQNpNoEREVglJPDn2kfGu+njk+cJpQkpyWyI0jqZK6UZdhKdYQ3XLwFpHVT8unI1+Dr5CBzru4YYaN3J6wTC5Yoqg/NU3/KAWeikkIUiuY7qvDLvHI4Lqg7leDcnIiuSbgJ4X9wHiK0dCIqSpXev diff --git a/roles/slave_tools/tasks/main.yml b/roles/slave_tools/tasks/main.yml index 618593c..e859cfe 100644 --- a/roles/slave_tools/tasks/main.yml +++ b/roles/slave_tools/tasks/main.yml @@ -11,11 +11,6 @@ fail: msg="You must have AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in your environment" when: aws_id.msg == "" or aws_secret.msg == "" -- name: Create user 'ubuntu' - authorized_key: user=ubuntu key="{{item}}" - with_file: - - Jenkins_aws_system.pem.pub -- authorized_key: user=ubuntu key='{{lookup("pipe", "cat /opt/build/hudson/authorized_keys/id_*.pub" )}}' - name: Create user 'jenkins' user: name=jenkins home=/opt/jenkins state=present shell=/bin/bash - file: path=/opt/jenkins/ state=directory owner=jenkins group=jenkins diff --git a/slave.yml b/slave-common.yml similarity index 90% rename from slave.yml rename to slave-common.yml index c379927..b175f60 100644 --- a/slave.yml +++ b/slave-common.yml @@ -12,5 +12,4 @@ roles: - slave_tools - slave_config_common - - slave_config_qa diff --git a/slave-priv.yml b/slave-priv.yml new file mode 100644 index 0000000..aa0fa76 --- /dev/null +++ b/slave-priv.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + gather_facts: no + user: ubuntu + become: yes + roles: + - slave_config_qapriv + diff --git a/slave-pub.yml b/slave-pub.yml new file mode 100644 index 0000000..e6d40d1 --- /dev/null +++ b/slave-pub.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + gather_facts: no + user: ubuntu + become: yes + roles: + - slave_config_qa + diff --git a/slavepriv.yml b/slavepriv.yml deleted file mode 100644 index 87d6a21..0000000 --- a/slavepriv.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- hosts: localhost - gather_facts: no - become: no - roles: - - slave_cache - -- hosts: all - gather_facts: no - user: ubuntu - become: yes - roles: - - slave_tools - - slave_config_common - - slave_config_qapriv -