Configure OBS instance to perform Clang builds

README.md

@@ -0,0 +1,17 @@
+# Open Build Service at irill8.siege.inria.fr
+
+This repository hosts salt states to provision our OBS intance at
+irill8.siege.inria.fr.
+
+For now, this OBS instance monitors the debian-devel-changes mailing list and
+triggers Clang builds for newly accepted packages
+
+## Adding new workers to the OBS instance
+
+To configure new workers to our current OBS instance hosted at
+irill8.siege.inria.fr, just set new salt slaves and provision them with
+`obs-common` and `obs-worker`.
+
+In this sense, the steps needed are: install the obs-worker package from Debian
+Stable and substitute the `/etc/default/obsworker` configuration file with the
+one provisioned by this repository.

obs-pillar/email-credentials.sls

@@ -0,0 +1,5 @@
+email-credentials:
+  lookup:
+    pass: password
+    user: username
+    hostname: example.com

obs-pillar/obs-hosts.sls

@@ -0,0 +1,3 @@
+obs-hosts:
+  lookup:
+    server: irill8.siege.inria.fr

obs-pillar/obs-osc.sls

@@ -0,0 +1,3 @@
+obs-osc:
+  lookup:
+    pass: opensuse

obs-pillar/top.sls

@@ -1,3 +1,6 @@
 base:
   '*':
     - obs-database
+    - obs-osc
+    - obs-hosts
+    - email-credentials

obs-server.sls

@@ -1,4 +1,21 @@
-install obs server packages:
+/etc/apt/sources.list:
+  file.managed:
+    - source: salt://obs-server/sources.list
+    - user: root
+    - group: root
+    - mode: 644
+
+obs:
+  host.present:
+    - ip: 127.0.0.1
+
+refresh_packages_db:
+  cmd.run:
+    - name: apt-get update -y
+    - onchanges:
+      - file: /etc/apt/sources.list
+
+install_obs_server_packages:
   pkg.installed:
     - pkgs:
       - obs-server
@@ -9,11 +26,11 @@ obs-api:
   pkg:
     - installed
   file.managed:
-     - name: /usr/share/obs/api/config/database.yml
-     - source: salt://llvm-obs/obs-conf/obs-api_database.yml
-     - template: jinja
+    - name: /usr/share/obs/api/config/database.yml
+    - source: salt://llvm-obs/obs-conf/obs-api_database.yml
+    - template: jinja
 
-install apache:
+install_apache:
   pkg.installed:
     - pkgs:
       - apache2
@@ -22,11 +39,141 @@ install apache:
       - libapache2-mod-xforward
       - memcached
 
-create self signed ssl for testing:
+create_self_signed_ssl_for_testing:
   cmd.script:
     - name: generate_ssl.sh
     - source: salt://llvm-obs/obs-scripts/generate_ssl.sh
 
-rake task setup:
+install_obs_build_from_backports:
+  pkg.latest:
+    - pkgs:
+      - obs-build
+    - fromrepo: stretch-backports
+
+install_libsolv_from_testing:
+  pkg.latest:
+    - pkgs:
+      - libsolv0
+      - libsolv-perl
+      - libsolvext0
+    - fromrepo: buster
+
+/usr/share/obs/api/Gemfile:
+  file.managed:
+    - source: salt://obs-server/Gemfile
+    - user: root
+    - group: root
+    - mode: 644
+
+{% if salt['grains.get']('api_setup') != 'done' %}
+rake_task_setup:
   cmd.run:
      - name: "bash /usr/share/obs/api/script/rake-tasks.sh setup"
+  grains.present:
+    - name: api_setup
+    - value: done
+
+restart_apache:
+  service.running:
+    - name: apache2
+    - enable: True
+    - watch:
+      - rake_task_setup
+{% endif %}
+
+start_obsservice:
+  service.running:
+    - name: obsservice
+    - enable: True
+
+/root/.oscrc:
+  file.managed:
+    - source: salt://obs-server/oscrc
+    - user: root
+    - group: root
+    - mode: 600
+    - template: jinja
+
+/tmp/obs_instance_configuration.xml:
+  file.managed:
+    - source: salt://obs-server/obs_instance_configuration.xml
+    - user: root
+    - group: root
+    - mode: 644
+
+/tmp/debian_unstable.xml:
+  file.managed:
+    - source: salt://obs-server/debian_unstable.xml
+    - user: root
+    - group: root
+    - mode: 644
+
+/tmp/debian_unstable.conf:
+  file.managed:
+    - source: salt://obs-server/debian_unstable.conf
+    - user: root
+    - group: root
+    - mode: 644
+
+/tmp/debian_clang.xml:
+  file.managed:
+    - source: salt://obs-server/debian_clang.xml
+    - user: root
+    - group: root
+    - mode: 644
+
+restart_obssrcserver:
+  service.running:
+    - name: obssrcserver
+    - enable: True
+    - watch:
+      - create_self_signed_ssl_for_testing
+
+set_obs_instance_configurations:
+  cmd.run:
+    - name: osc api /configuration -T /tmp/obs_instance_configuration.xml
+
+create_debian_unstable_project:
+  cmd.run:
+    - name: osc meta prj Debian:Unstable -F /tmp/debian_unstable.xml
+
+configure_debian_unstable_project:
+  cmd.run:
+    - name: osc meta prjconf Debian:Unstable -F /tmp/debian_unstable.conf
+
+create_debian_clang_project:
+  cmd.run:
+    - name: osc meta prj Debian:Unstable:Clang -F /tmp/debian_clang.xml
+
+/usr/local/bin/trigger_clang_build:
+  file.managed:
+    - source: salt://obs-server/trigger_clang_build
+    - user: root
+    - group: root
+    - mode: 755
+
+build_obs_clang_build_package:
+  cmd.run:
+    - name: trigger_clang_build obs-service-clang-build
+
+/usr/local/bin/check_new_uploads:
+  file.managed:
+    - source: salt://obs-server/check_new_uploads
+    - user: root
+    - group: root
+    - mode: 755
+  cron.present:
+    - user: root
+    - minute: 15
+    - hour: '*/2'
+
+/tmp/obs_service_clang_build_meta.xml:
+  file.managed:
+    - source: salt://obs-server/obs_service_clang_build_meta.xml
+    - user: root
+    - group: root
+    - mode: 644
+
+allow_obs_service_clang_build_usage:
+  cmd.run:
+    - name: osc meta pkg Debian:Unstable:Clang obs-service-clang-build -F /tmp/obs_service_clang_build_meta.xml

obs-server/Gemfile

@@ -0,0 +1,93 @@
+# Edit this Gemfile to bundle your application's dependencies.
+# This preamble is the current preamble for Rails 3 apps; edit as needed.
+source 'https://rubygems.org'
+
+gem 'rails', '~> 4.2.2'
+gem 'actionmailer'
+
+# as our database
+gem 'mysql2'
+# as requirement for activexml
+gem 'nokogiri'
+# for delayed tasks
+gem 'delayed_job_active_record', '>= 4.0.0'
+# as JSON library - the default json conflicts with activerecord (by means of vice-versa monkey patching)
+gem 'yajl-ruby'
+# to search the database
+gem 'thinking-sphinx', '> 3.1'
+# to paginate search results
+gem 'kaminari'
+# as abstract HTML of the bratwurst 'theme'
+gem 'haml'
+# to avoid tilt downgrade
+gem 'tilt', '>= 1.4.1'
+# to use markdown in the comment system
+gem 'redcarpet'
+# for nested attribute forms
+gem 'cocoon'
+# for activerecord lists. Used for AttribValues
+gem 'acts_as_list'
+# to parse a XML string into a ruby hash
+gem 'xmlhash', '>=1.3.6'
+# to escape HTML (FIXME: do we still use this?)
+gem 'escape_utils'
+# to sanitize HTML/CSS
+gem 'sanitize'
+# as authorization system
+gem "pundit"
+#
+gem 'responders', '~> 2.0'
+# needed for travis-ci.org, must be global for scripts
+gem 'bundler'
+# for threaded comments
+gem 'acts_as_tree'
+# js plotting (OBS monitor)
+gem 'flot-rails'
+
+group :development, :production do
+  # to have the delayed job daemon
+  gem 'daemons'
+  # as memcache client
+  gem 'dalli', require: false
+  # to document ruby code
+  gem 'rdoc'
+  # to not rely on cron+rake
+  gem 'clockwork', '>= 0.7'
+  # as interface to LDAP
+  gem 'ruby-ldap', require: false
+end
+
+group :production do
+  # if you have an account, it can be configured by
+  # placing a config/newrelic.yml
+  # be aware about the non-OSS license
+#  gem 'newrelic_rpm'
+end
+
+# Gems used only for assets and not required in production environments by default.
+group :assets do
+  # for minifying CSS
+  gem 'cssmin', '>= 1.0.2'
+  # for minifying JavaScript
+  gem 'uglifier', '>= 1.2.2'
+  # to use sass in the asset pipeline
+  gem 'sass-rails', '~> 5.0.1'
+  # assets for jQuery DataTables
+  gem 'jquery-datatables-rails', '1.12.2', path: "vendor/gems/jquery-datatables-rails-1.12.2"
+  # assets for the text editor
+  gem 'codemirror-rails'
+  # assets for jQuery tokeninput
+  gem 'rails_tokeninput', '>= 1.6.1.rc1'
+  # to create our sprite images/stylesheets
+  gem 'sprite-factory', '>= 1.5.2'
+  # to read and write PNG images
+  gem 'chunky_png'
+  # assets for jQuery and jQuery-ujs
+  gem 'jquery-rails'
+  # assets for jQuery-ui
+  gem 'jquery-ui-rails', '>= 4.2.1' # version 5 needs henne's new webui
+  # assets for the bootstrap front-end framework. Used by the bratwurst theme
+  # gem 'bootstrap-sass-rails'
+  # assets for font-awesome vector icons
+  gem "font-awesome-rails"
+end

obs-server/check_new_uploads

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import imaplib
+import email
+import subprocess
+import time
+
+TRIGGER_BIN_PATH = '/usr/local/bin/trigger_clang_build'
+
+# Hours to wait before triggering a build;
+# this allows propagation time so the package is available in source mirrors.
+COOLDOWN = 6
+
+# Mail server IMAP information and credentials;
+# this must be an email subscribed to the debian-devel-changes mailing list.
+IMAP_SERVER = "{{ salt['pillar.get']('email-credentials:lookup:hostname') }}"
+USERNAME = "{{ salt['pillar.get']('email-credentials:lookup:user') }}"
+PASSWORD = "{{ salt['pillar.get']('email-credentials:lookup:pass') }}"
+
+
+def parse_message(msg):
+    lines = msg.split('\n')
+    mapping = {
+        'Source': 'name',
+        'Version': 'version',
+        'Architecture': 'arch',
+        'Distribution': 'dist',
+        'Maintainer': 'maintainer'
+    }
+    pkg = {}
+    for line in lines:
+        if len(mapping) == 0:
+            break
+        for field, target in mapping.items():
+            if line.startswith('%s: ' % field):
+                val = line[len(field) + 2:]
+                pkg[target] = val.strip()
+                del mapping[field]
+                break
+    return pkg
+
+
+def trigger_build(pkg):
+    subprocess.run([TRIGGER_BIN_PATH, pkg['name']])
+
+
+subprocess.run(['apt', 'update'])
+list_id = '<debian-devel-changes.lists.debian.org>'
+M = imaplib.IMAP4_SSL(IMAP_SERVER)
+M.login(USERNAME, PASSWORD)
+M.select()
+typ, data = M.search(None, 'UNSEEN')
+for num in data[0].split():
+    typ, data = M.fetch(num, '(RFC822)')
+    parsed_email = email.message_from_bytes(data[0][1])
+
+    # We want to give some time so the upload can propagate to the mirrors
+    email_time = email.utils.parsedate_tz(parsed_email['date'])
+    email_time_utc = email.utils.mktime_tz(email_time)
+    upload_time_limit = time.time() - COOLDOWN*60*60
+    if email_time_utc > upload_time_limit:
+        # Too recent to be parsed
+        M.store(num, '-FLAGS', '\\SEEN')
+        continue
+
+    if(parsed_email['subject'].startswith('Accepted')
+       and parsed_email['list-id'] == list_id):
+        pkg = parse_message(parsed_email.get_payload())
+        if (pkg['dist'] in ['unstable', 'sid']) and 'source' in pkg['arch']:
+            trigger_build(pkg)
+    # Uncomment the following lines if you want to remove the email afterwards
+    # M.store(num, '+FLAGS', '\\Deleted')
+# M.expunge()
+M.close()
+M.logout()