Added plugins support, including for MQTT

Signed-off-by: Fabian Arrotin <arrfab@centos.org>

defaults/main.yml

@@ -63,6 +63,16 @@ koji_admin_client: True
 koji_admin_user: admin
 koji_admin_pem: admin.pem
 
+# Kojihub plugins
+koji_hub_plugins: True
+koji_hub_plugins_list: []
+
+# Now the plugins settings, if needed
+koji_hub_plugin_mqtt_host: mqtt.dev.centos.org
+koji_hub_plugin_mqtt_topic: koji
+koji_hub_plugin_mqtt_excluded_tags: 
+  - sclo-testing-tag
+
 # Zabbix/monitoring part
 kojihub_zabbix_templates:
   - Template CentOS Koji Hub

files/plugins/centmsg.py

@@ -0,0 +1,162 @@
+# Koji callback sent to CentOS mqtt                                            
+#                                                                              
+# Adapted from https://gitlab.cern.ch/linuxsupport/rpms/koji-hub-plugins-cern/blob/master/src/mash.py
+#                                                                              
+# License: GPLv2                                                               
+# Authors:                                                                     
+#     Alex Iribarren <alex.iribarren@cern.ch> (original script)                
+#     Thomas (dot) Oulevey (at) cern (dot) ch (mqtt version)
+
+import koji
+from koji import PluginError
+from koji.context import context
+from koji.plugin import callback, ignore_error
+import kojihub
+import ConfigParser
+import logging
+import base64, json
+import os
+
+# mqtt client
+import paho.mqtt.client as mqtt
+
+CONFIG_FILE = '/etc/koji-hub/plugins/centmsg.conf'
+PLUGIN_NAME = 'koji.plugin.centmsg'
+DEFAULT_ARCHES = 'x86_64'
+
+config = None
+tagCache = {}
+
+def get_config():
+    global config
+    if config:
+        return config
+
+    config = ConfigParser.SafeConfigParser()
+    config.read(CONFIG_FILE)
+
+    if not config.has_section('centmsg'):
+        config.add_section('centmsg')
+    if not config.has_option('centmsg', 'host'):
+        logging.getLogger(PLUGIN_NAME).error('No mqtt host specified in config file!')
+        return None
+    if not config.has_option('centmsg', 'port'):
+        logging.getLogger(PLUGIN_NAME).error('No mqtt port specified in config file!')
+        return None
+    if not config.has_option('centmsg', 'topic'):
+        logging.getLogger(PLUGIN_NAME).error('No mqtt topic specified in config file!')
+        return None
+    if not config.has_option('centmsg', 'ca_cert'):
+        logging.getLogger(PLUGIN_NAME).error('No mqtt cacert specified in config file!')
+        return None
+    if not config.has_option('centmsg', 'tls_cert'):
+        logging.getLogger(PLUGIN_NAME).error('No mqtt tls_cert specified in config file!')
+        return None
+    if not config.has_option('centmsg', 'tls_key'):
+        logging.getLogger(PLUGIN_NAME).error('No mqtt tls_key specified in config file!')
+        return None
+    if not config.has_option('centmsg', 'tls_insecure'):
+        config.set('centmsg' 'tls_insecure', 'False')
+    if not config.has_option('centmsg', 'tls_version'):
+        config.set('centmsg' 'tls_version', '2')
+    if not config.has_option('centmsg', 'exclude_tags'):
+        config.set('centmsg', 'exclude_tags', '')
+
+    return config
+
+def mqtt_on_publish(client,userdata,result):
+  pass 
+
+def _dispatch_on_topic(payload):
+    logger = logging.getLogger(PLUGIN_NAME)
+
+    config = get_config()
+    if not config:
+        raise PluginError('Unable to use the bus, config not found')
+
+    if not payload['tag']:
+        logger.info('No tag specified')
+        return None
+
+    exclude_tags = config.get('centmsg', 'exclude_tags')
+    if exclude_tags:
+      exclude_tags = [x.strip() for x in exclude_tags.split(',')]
+    else:
+      exclude_tags = []
+
+    if payload['tag'] in exclude_tags:
+        logger.info('Tag %s excluded' % payload['tag'])
+        return None
+
+    mqtt_host          = config.get('centmsg', 'host')
+    mqtt_port          = config.get('centmsg', 'port')
+    mqtt_topic         = config.get('centmsg', 'topic')
+    mqtt_cacert        = config.get('centmsg', 'ca_cert')
+    mqtt_tls_cert      = config.get('centmsg', 'tls_cert')
+    mqtt_tls_key       = config.get('centmsg', 'tls_key')
+    mqtt_tls_insecure  = config.get('centmsg', 'tls_insecure')
+    mqtt_tls_version   = config.get('centmsg', 'tls_version')
+
+    # Connect to the bus
+    try:
+        client = mqtt.Client()
+    except Exception as e:
+        logger.error('mqtt client error: %s' % e.message) 
+    client.tls_set(ca_certs=mqtt_cacert, certfile=mqtt_tls_cert, keyfile=mqtt_tls_key, tls_version=2)
+
+    client.tls_insecure_set('False')
+    try:
+        client.on_publish = mqtt_on_publish
+        client.connect(mqtt_host,mqtt_port)
+    except Exception as e:
+        logger.error('mqtt connection error: %s' % e.message)
+
+    # Publish payload to the bus
+    #
+    ret = client.publish(mqtt_topic, json.dumps(payload))
+
+    # Disconnect from the bus
+    client.disconnect()
+
+    return ret
+
+def _get_build_target(task_id):
+    try:
+        task = kojihub.Task(task_id)
+        info = task.getInfo(request=True)
+        request = info['request']
+        if info['method'] in ('build', 'maven'):
+            # request is (source-url, build-target, map-of-other-options)
+            if request[1]:
+                return kojihub.get_build_target(request[1])
+        elif info['method'] == 'winbuild':
+            # request is (vm-name, source-url, build-target, map-of-other-options)
+            if request[2]:
+                return kojihub.get_build_target(request[2])
+    except Exception as e:
+        logger.error('Exception: %s', e)
+
+    return None
+
+
+@callback('postTag', 'postUntag')
+#@ignore_error
+def centmsg(cbtype, *args, **kws):
+    logger = logging.getLogger(PLUGIN_NAME)
+    logger.debug('Called the %s callback, args: %s; kws: %s', cbtype, str(args), str(kws))
+
+    tag = kws['tag']['name']
+    build_task_id = kws['build']['task_id']
+
+    build_target = _get_build_target(build_task_id)
+    logger.debug('Build target: %s', build_target)
+
+    arches = DEFAULT_ARCHES
+    if build_target:
+        build_tag = kojihub.get_tag(build_target['build_tag_name'])
+        arches = build_tag['arches']
+
+    payload = { 'action': cbtype, 'tag': tag, 'arches': arches }
+    job = _dispatch_on_topic(payload)
+    if job:
+        logger.info('Sending payload: %s to mqtt - ret code: %s' % (payload, job))

tasks/main.yml

@@ -19,7 +19,6 @@
   tags:
     - repo
 
-
 - name: Ensuring we have some pkgs installed
   yum:
     name: "{{ item }}"
@@ -33,6 +32,7 @@
     - git
     - gnupg2
     - repoview
+    - python-paho-mqtt
 
 - include_tasks: storage.yml
 - include_tasks: tls.yml
@@ -70,6 +70,9 @@
 
 - include_tasks: koji-admin.yml
   when: koji_admin_client
+- include_tasks : plugins.yml
+  tags:
+    - plugins
 
 - include_tasks: kojira.yml
   when: koji_kojira

tasks/plugins.yml

@@ -0,0 +1,15 @@
+- name: Deploying enabled plugins
+  copy:
+    src: "plugins/{{ item }}.py"
+    dest: "/usr/lib/koji-hub-plugins/{{ item }}.py"
+  with_items: "{{ koji_hub_plugins_list }}"
+  tags:
+    - plugins
+
+- name: Configuring enabled plugins
+  template:
+    src: "koji-hub/plugins/{{ item }}.conf.j2"
+    dest: "/etc/koji-hub/plugins/{{ item }}.conf"
+  with_items: "{{ koji_hub_plugins_list }}"
+  tags:
+    - plugins

tasks/selinux.yml

@@ -5,5 +5,6 @@
     state: yes
   with_items:
     - httpd_can_network_connect_db
+    - httpd_can_network_connect
     - allow_httpd_anon_write
     - httpd_use_nfs

templates/koji-hub/hub.conf.j2

@@ -64,6 +64,9 @@ DisableNotifications = True
 # PluginPath = /usr/lib/koji-hub-plugins
 ## A space-separated list of plugins to load
 # Plugins = echo
+{% if koji_hub_plugins %}
+Plugins = {% for plugin in koji_hub_plugins_list %}{{ plugin }} {% endfor %}
+{% endif %}
 
 ## If KojiDebug is on, the hub will be /very/ verbose and will report exception
 ## details to clients for anticipated errors (i.e. koji's own exceptions --

templates/koji-hub/plugins/centmsg.conf.j2

@@ -0,0 +1,11 @@
+[centmsg]
+host = {{ koji_hub_plugin_mqtt_host }}
+port = 8883
+tls_cert = /etc/pki/koji/{{ koji_admin_pem }}
+tls_key = /etc/pki/koji/{{ koji_admin_pem }}
+ca_cert = /etc/pki/koji/{{ koji_hub_cacert }}
+topic = {{ koji_hub_plugin_mqtt_topic }}
+tls_insecure = False
+tls_version = 2
+exclude_tags = {% for tag in koji_hub_plugin_mqtt_excluded_tags %}{{ tag }}{%- if not loop.last -%},{% endif %}{% endfor %}
+