Add web service for tracking intermittent failure instances

buildbot/master/files/config/environments.py

@@ -1,3 +1,4 @@
+from buildbot.plugins import util
 from passwords import GITHUB_DOC_TOKEN, GITHUB_HOMEBREW_TOKEN
 from passwords import S3_UPLOAD_ACCESS_KEY_ID, S3_UPLOAD_SECRET_ACCESS_KEY
 
@@ -49,6 +50,7 @@ def without(self, to_unset):
 
 build_common = Environment({
     'RUST_BACKTRACE': '1',
+    'BUILD_MACHINE': str(util.Property('slavename')),
 })
 
 build_windows_msvc = build_common + Environment({

docs/intermittent-failure-tracker.md

@@ -0,0 +1,13 @@
+# Intermittent Failure Tracker service
+
+This service runs a flask server that serves two purposes:
+
+1. an API endpoint for Servo's CI builders to report
+instances of intermittent failures that are encountered
+
+2. an HTML frontend to explore trends in
+intermittent failures that have been reported.
+
+See the project's [README](https://github.com/servo/intermittent-failure-tracker/blob/master/README.md) for more details about interacting with the service,
+and [this Servo pull request](https://github.com/servo/servo/pull/16946)
+for more details about how failures are reported.

intermittent-failure-tracker/files/config.json

@@ -0,0 +1 @@
+{"port": 5002}

intermittent-failure-tracker/files/tracker.conf

@@ -0,0 +1,11 @@
+exec /home/servo/intermittent-failure-tracker/_venv/bin/intermittent_failure_tracker
+
+setuid servo
+setgid servo
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on runlevel [016]
+
+env HOME=/home/servo
+
+chdir /home/servo/intermittent-failure-tracker

intermittent-failure-tracker/init.sls

@@ -0,0 +1,48 @@
+{% from tpldir ~ '/map.jinja' import tracker %}
+
+include:
+  - python
+
+intermittent-failure-tracker:
+  virtualenv.managed:
+    - name: /home/servo/intermittent-failure-tracker/_venv
+    - venv_bin: virtualenv-3.5
+    - python: python3
+    - system_site_packages: False
+    - require:
+      - pkg: python3
+      - pip: virtualenv
+  pip.installed:
+    - pkgs:
+      - git+https://github.com/servo/intermittent-failure-tracker@{{ tracker.rev }}
+    - bin_env: /home/servo/intermittent-failure-tracker/_venv
+    - require:
+      - virtualenv: intermittent-failure-tracker
+  {% if grains.get('virtual_subtype', '') != 'Docker' %}
+  service.running:
+    - enable: True
+    - name: failure-tracker
+    - require:
+      - pip: intermittent-failure-tracker
+    - watch:
+      - file: /home/servo/intermittent-failure-tracker/config.json
+      - file: /etc/init/failure-tracker.conf
+  {% endif %}
+
+/home/servo/intermittent-failure-tracker/config.json:
+  file.managed:
+    - source: salt://{{ tpldir }}/files/config.json
+    - template: jinja
+    - user: servo
+    - group: servo
+    - mode: 644
+
+/etc/init/failure-tracker.conf:
+  file.managed:
+    - source: salt://{{ tpldir }}/files/tracker.conf
+    - user: root
+    - group: root
+    - mode: 644
+    - require:
+      - pip: intermittent-failure-tracker
+      - file: /home/servo/intermittent-failure-tracker/config.json

intermittent-failure-tracker/map.jinja

@@ -0,0 +1,5 @@
+{%
+  set tracker = {
+    'rev': '37ef73bb0059e0555a903e758dae8006c46a9916'
+  }
+%}

nginx/default

@@ -16,5 +16,8 @@ server {
   location /intermittent-tracker/ {
     proxy_pass http://localhost:5000/;
   }
+  location /intermittent-failure-tracker/ {
+    proxy_pass http://localhost:5002/;
+  }
 }
 

top.sls

@@ -43,5 +43,6 @@ base:
     - buildbot.master
     - homu
     - intermittent-tracker
+    - intermittent-failure-tracker
     - nginx
     - salt.master