Run promoter sequancially between releases

Simplify the promoter removing the lock and running it
sequencially one release after another.

It also start it up as a systemd service unit.

Change-Id: I5ac9daed9ed4a3957b8761bd05b647fb0244dc84

ci-scripts/dlrnapi_promoter/dlrn-promoter.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -x
+
+#Activate virtualenv
+source ~/promoter_venv/bin/activate
+
+while true; do
+
+    #fetch the latest ci-config
+    cd ~/ci-config; git pull >/dev/null
+
+    #fetch the latest dlrnapi_client and dependencies
+    pip install -U -r ~/ci-config/ci-scripts/dlrnapi_promoter/requirements.txt
+
+    # Source secrets
+    source ~/registry_secret
+    source ~/dlrnapi_secret
+
+    # promoter script for the master branch
+    /usr/bin/timeout --preserve-status -k 120m 115m python ~/ci-config/ci-scripts/dlrnapi_promoter/dlrnapi_promoter.py ~/ci-config/ci-scripts/dlrnapi_promoter/config/master.ini
+
+    # promoter script for the pike branch
+    /usr/bin/timeout --preserve-status -k 120m 115m python ~/ci-config/ci-scripts/dlrnapi_promoter/dlrnapi_promoter.py ~/ci-config/ci-scripts/dlrnapi_promoter/config/pike.ini
+
+    # promoter script for the ocata branch
+    /usr/bin/timeout --preserve-status -k 120m 115m python ~/ci-config/ci-scripts/dlrnapi_promoter/dlrnapi_promoter.py ~/ci-config/ci-scripts/dlrnapi_promoter/config/ocata.ini
+
+    # promoter script for the queens branch
+    /usr/bin/timeout --preserve-status -k 120m 115m python ~/ci-config/ci-scripts/dlrnapi_promoter/dlrnapi_promoter.py ~/ci-config/ci-scripts/dlrnapi_promoter/config/queens.ini
+
+    # Sleep 10 minutes
+    sleep 600
+done

ci-scripts/dlrnapi_promoter/dlrnapi_promoter.py

@@ -276,27 +276,10 @@ def promote_all_links(api, promote_from, job_reqs, dry_run, release, latest_hash
                                 current_name, promote_name, new_hashes)
 
 
-# Use atomic abstract socket creation as process lock
-# no pid files to deal with
-def get_lock(process_name):
-    logger = logging.getLogger('promoter')
-    # Without holding a reference to our socket somewhere it gets garbage
-    # collected when the function exits
-    get_lock._lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
-
-    try:
-        get_lock._lock_socket.bind('\0' + process_name)
-        logger.debug('No other promoters running. Acquired lock and continuing with promotion')
-    except socket.error:
-        logger.error('Unable to acquire lock. Another promoter process is running. Aborting.')
-        sys.exit(1)
-
-
 def promoter(config):
     logger = logging.getLogger('promoter')
 
     release = config.get('main', 'release')
-    get_lock('promoter')
 
     logger.info('STARTED promotion process for release: %s', release)
 

ci-scripts/infra-setup/roles/promoter/files/dlrn-promoter.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Tripleo CI DLRN promoter
+Requires=network.target
+After=syslog.target network.target
+
+[Service]
+ExecStart=/home/centos/ci-config/ci-scripts/dlrnapi_promoter/dlrn-promoter.sh
+User=centos
+Restart=always

ci-scripts/infra-setup/roles/promoter/files/dlrn-promoter.telegraf.conf

@@ -0,0 +1,8 @@
+[[inputs.procstat]]
+    pattern = "^python.*dlrnapi_promoter.py .*master.*"
+[[inputs.procstat]]
+    pattern = "^python.*dlrnapi_promoter.py .*queens.*"
+[[inputs.procstat]]
+    pattern = "^python.*dlrnapi_promoter.py .*pike.*"
+[[inputs.procstat]]
+    pattern = "^python.*dlrnapi_promoter.py .*ocata.*"

ci-scripts/infra-setup/roles/promoter/tasks/main.yml

@@ -21,6 +21,15 @@
     fstype: xfs
     state: mounted
 
+- name: Install influxdb repo to get telegraf
+  yum_repository:
+    name: influxdb
+    description: InfluxDB Repository - RHEL \$releasever
+    baseurl: https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable
+    enabled: yes
+    gpgcheck: yes
+    gpgkey: https://repos.influxdata.com/influxdb.key
+
 - name: Install required packages
   yum:
     name: '{{ item }}'
@@ -30,6 +39,7 @@
     - docker-python
     - httpd
     - python-virtualenv
+    - telegraf
 
 - name: Create a directory for the promoter logs
   file:
@@ -151,15 +161,30 @@
         requirements: ~/ci-config/ci-scripts/dlrnapi_promoter/requirements.txt
         virtualenv: ~/promoter_venv
 
-    - name: Add a cron jobs to run the promotion script
-      cron:
-        name: 'promoter script for the {{ item.release }} branch'
-        minute: '{{ item.minute }}'
-        # TODO(gcerami) I think the job command deserves a script on its own
-        job: 'source ~/registry_secret; source ~/dlrnapi_secret; source ~/promoter_venv/bin/activate; /usr/bin/timeout --preserve-status -k 120m 115m python ~/ci-config/ci-scripts/dlrnapi_promoter/dlrnapi_promoter.py ~/ci-config/ci-scripts/dlrnapi_promoter/config/{{ item.release }}.ini'
-      with_items:
-        - { release: 'master', minute: '1,11,21,31,41,51' }
-        - { release: 'queens', minute: '4,14,24,34,44,54' }
-        - { release: 'pike', minute: '2,12,22,32,42,52' }
-        - { release: 'ocata', minute: '3,13,23,33,43,53' }
+- name: Install promoter service
+  copy:
+    src: dlrn-promoter.service
+    dest: /etc/systemd/system/
+    mode: '0644'
+    owner: 'root'
+    group: 'root'
+
+- name: Start and enable promoter
+  service
+    name: dlrn-promoter
+    state: started
+    enabled: yes
 
+- name: Copy telegraf config for promoter
+  copy:
+    src: dlrn-promoter.telegraf.conf
+    dest: /etc/telegraf/telegraf.d/dlrn-promoter.conf
+    mode: '0644'
+    owner: 'root'
+    group: 'root'
+
+- name: Start and enable telegraf
+  service
+    name: telegraf
+    state: started
+    enabled: yes