docs: Add a chapter about patch testing with patchwork

It's time to explain the new capabilities of patchwork, namely exposing
the list of new series sent to the mailing-list and the ability to store
test results or link to them.

Signed-off-by: Damien Lespiau <damien.lespiau@intel.com>

docs/examples/testing-checkpatch.py

@@ -0,0 +1,134 @@
+#!/bin/env python2
+# coding=utf-8
+import fileinput
+import json
+import mailbox
+import os
+import re
+import requests
+import subprocess
+import tempfile
+
+# config
+PATCHWORK_URL = 'http://patchwork.freedesktop.org'
+KERNEL_PATH = '/home/damien/gfx/sources/linux'
+
+API_URL = PATCHWORK_URL + '/api/1.0'
+
+
+class TestState(object):
+    PENDING = 0
+    SUCCESS = 1
+    WARNING = 2
+    FAILURE = 3
+
+
+class Test(object):
+    '''A test on a series.'''
+
+    def __init__(self, series, revision):
+        self.series = series
+        self.revision = revision
+        self.state = TestState.PENDING
+        self.summary = ''
+        self.url = None
+
+    def merge_patch_result(self, state, summary):
+        if state > self.state:
+            self.state = state
+        self.summary += summary
+
+    def post_result(self):
+        state_names = ['pending', 'success', 'warning', 'failure']
+        cmd = subprocess.Popen(['git-pw',
+                                '-C', KERNEL_PATH,
+                                'post-result',
+                                str(self.series),
+                                '--revision', str(self.revision),
+                                self.TEST_NAME,
+                                state_names[self.state],
+                                '--summary', self.summary])
+        cmd.wait()
+
+
+class CheckpatchTest(Test):
+    CHECKPATCH = KERNEL_PATH + '/scripts/checkpatch.pl'
+    TEST_NAME = 'UK.CI.checkpatch.pl'
+    IGNORE_LIST = ['SPLIT_STRING', 'COMPLEX_MACRO', 'GIT_COMMIT_ID',
+                   'COMMIT_LOG_LONG_LINE', 'BLOCK_COMMENT_STYLE',
+                   'FILE_PATH_CHANGES']
+
+    def _counts(self, results):
+        counts = [0, 0]
+        error = re.compile(r'^ERROR:')
+        warning = re.compile(r'^WARNING')
+        for line in iter(results.splitlines()):
+            if error.search(line):
+                counts[0] += 1
+            elif warning.search(line):
+                counts[1] += 1
+        return counts
+
+    def _run(self, mail):
+        cmd = subprocess.Popen([self.CHECKPATCH,
+                                '--mailback', '--no-summary',
+                                '--max-line-length=100',
+                                '--ignore', ','.join(self.IGNORE_LIST),
+                                '-'],
+                               stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                               stderr=subprocess.STDOUT)
+        (stdout, _) = cmd.communicate(input=mail.as_string())
+        state = TestState.SUCCESS
+        if cmd.returncode != 0:
+            (n_errors, n_warnings) = self._counts(stdout)
+            if n_errors > 0:
+                state = TestState.FAILURE
+            elif n_warnings > 0:
+                state = TestState.WARNING
+        if stdout and len(stdout) > 1:
+            stdout = '\n' + stdout + '\n'
+        return (state, stdout)
+
+    def process_patch(self, mail):
+        (state, stdout) = self._run(mail)
+        header = u'  • Testing %s\n' % mail.get('Subject').replace('\n', '')
+        self.merge_patch_result(state, header + stdout)
+
+
+class TestRunner(object):
+    def __init__(self, test_class):
+        self.test_class = test_class
+
+    def _process_event(self, event):
+        # Retrieves the mbox file of a series and run process_patch() for each
+        # mail
+        (series, revision) = (event['series'], event['parameters']['revision'])
+        mbox_url = API_URL + ('/series/%d/revisions/%d/mbox/' %
+                              (series, revision))
+
+        print('== Running %s on series %d v%d' % (self.test_class.__name__,
+                                                  series, revision))
+
+        test = self.test_class(series, revision)
+
+        r = requests.get(mbox_url)
+        fd, path = tempfile.mkstemp()
+        try:
+            with os.fdopen(fd, 'w+') as tmp:
+                tmp.write(r.content)
+            for mail in mailbox.mbox(path):
+                test.process_patch(mail)
+        finally:
+            os.remove(path)
+
+        test.post_result()
+
+    def run(self):
+        # process events from stdin, one at a time
+        for line in fileinput.input():
+            self._process_event(json.loads(line))
+
+
+if __name__ == '__main__':
+    runner = TestRunner(CheckpatchTest)
+    runner.run()

docs/examples/testing-show-series.py

@@ -0,0 +1,9 @@
+#!/bin/python
+import fileinput
+import json
+
+for line in fileinput.input():
+   event = json.loads(line)
+   series = event['series']
+   revision = event['parameters']['revision']
+   print("series %d (rev %d)" % (series, revision))

docs/index.rst

@@ -14,6 +14,7 @@ Contents:
    intro
    installation
    manual
+   testing
    rest
    development
 

docs/manual.rst

@@ -147,6 +147,8 @@ cover letter subjects as new revisions:
   |                                 | - Awesome feature, take 6  |
   +---------------------------------+----------------------------+
 
+.. _git-pw:
+
 |git-pw|
 --------
 

docs/rest.rst

@@ -163,6 +163,8 @@ A project is merely one of the projects defined for this patchwork instance.
             "webscm_url": ""
         }
 
+.. _rest-events:
+
 Events
 ~~~~~~
 

docs/symbols

@@ -1,7 +1,12 @@
+.. |CI| replace:: :abbr:`CI (Continuous Integration)`
+.. |UI| replace:: :abbr:`UI (User Interface)`
 .. |git| replace:: :command:`git`
 .. |git-pw| replace:: :command:`git-pw`
+.. |git pw poll-events| replace:: :command:`git pw poll-events`
+.. |git pw post-result| replace:: :command:`git pw post-result`
 .. |git send-email| replace:: :command:`git send-email`
 .. |pip| replace:: :command:`pip`
 
 .. _diff: https://en.wikipedia.org/wiki/Diff_utility
 .. _repository: https://github.com/dlespiau/patchwork
+.. _requests: http://docs.python-requests.org/en/master/

docs/testing.rst

@@ -0,0 +1,195 @@
+Testing with Patchwork
+======================
+
+Patchwork can be used in conjunction with tests suites to build a |CI| system.
+
+
+Flow
+----
+
+
+Patches sent to the mailing list are grouped into *series* by Patchwork which,
+then, exposes *events* corresponding to the mailing-list activity. Listening to
+those *events*, one or more testing infrastructure(s) can retrieve the patches,
+test them and post test results back to Patchwork to display them in the web
+|UI|.  Optionally, Patchwork can send those test results back to the user
+and/or mailing-list.
+
+The following diagram describes that flow:
+
+.. image:: images/testing-ci-flow.png
+   :name: CI flow
+
+Series and Revisions
+~~~~~~~~~~~~~~~~~~~~
+
+Details about steps **1** and **2** can be found in :ref:`submitting-patches`.
+
+Polling for events
+~~~~~~~~~~~~~~~~~~
+
+Step **3** is Patchwork exposing new series and new series revisions appearing on
+the mailing list through the ``series-new-revision`` :ref:`event <rest-events>`.
+This event is created when Patchwork has seen all patches that are part of a
+new series/revision, so the API user can safely start processing the new series
+as soon as they notice the new event, which, for now, is done polling the
+:http:get:`/events/</api/1.0/projects/(string: linkname)/events/>` entry point.
+
+To poll for new events, the user can use the ``since`` :http:method:`get`
+parameter to ask for events since the last query. The time stamp to give to
+that ``since`` parameter is the ``event_time`` of the last event seen.
+
+Testing
+~~~~~~~
+
+Step **4** is, of course, where the testing happens.
+
+Test results
+~~~~~~~~~~~~
+
+Once done, results are posted back to Patchwork in step **5** with the
+:http:post:`/test-results/</api/1.0/series/(int: series_id)/revisions/(int: version)/test-results/>` entry point.
+
+One note on the intention behind the ``pending`` state: if running the test(s)
+takes a long time, it's a good idea to mark the test results as ``pending`` as
+soon as the ``series-new-revision`` event has been detected to indicate to the
+user their patches have been picked up for testing.
+
+Email reports
+~~~~~~~~~~~~~
+
+Finally, step **6**, the test results can be communicated back by mail to the
+submitter. By default, Patchwork will not send any email, that's to allow test
+scripts authors to develop without the risk of sending confusing emails to
+people.
+
+The test result emailing is configurable per test, identified by a unique tuple
+(``project``, ``test_name``). That configuration is done using the Django
+administration interface. The first parameter to configure is the email
+recipient(s):
+
+**none**
+  No test result email should be sent out (default).
+
+**submitter**
+  Test result emails are sent to the patch submitter.
+
+**mailing list**
+  Test result emails are sent to the patch submitter with the project
+  mailing-list in Cc.
+
+**recipient list**
+  Test result emails are sent to the list of email addresses specified in the
+  ``Mail To: list`` field.
+
+The ``Mail To: list`` and ``Mail Cc list`` are list of addresses that will be
+appended to the ``To:`` and ``Cc:`` fields.
+
+When the test is configured to send emails, the *when to send* can be tweaked
+as well:
+
+**always**
+  Always send an email, disregarding the status of the test result.
+
+**on failure**
+  Only send an email when the test has some warnings or errors.
+
+
+|git-pw| helper commands
+------------------------
+
+
+To interact with Patchwork, the REST API can be used directly with any language
+and an HTTP library. For python, requests_ is a winning choice and I'd have a
+look at the `git-pw source code`__.
+
+:ref:`git-pw` also provides a couple of commands that can help with writing
+test scripts without resorting to using the REST API.
+
+.. __: https://github.com/dlespiau/patchwork/blob/master/git-pw/git-pw
+
+|git pw poll-events|
+~~~~~~~~~~~~~~~~~~~~
+
+|git pw poll-events| will print events since the last invocation of this
+command. The output is one event per line as a JSON object, oldest event first.
+:command:`poll-events` stores the time stamp of the last event seen in a file
+called :file:`.git-pw.$project.poll.timestamp`.
+
+``--since`` can be used to override the last seen time stamp and ask for all the
+events since a specific date::
+
+    $ git pw poll-events --since=2016-02-12
+    {"series": 3324, "parameters": {"revision": 1}, "name": "series-new-revision", ... }
+    {"series": 3304, "parameters": {"revision": 3}, "name": "series-new-revision", ... }
+    {"series": 3072, "parameters": {"revision": 2}, "name": "series-new-revision", ... }
+    {"series": 3344, "parameters": {"revision": 1}, "name": "series-new-revision", ... }
+
+As shown, |git pw poll-events| prints JSON objects on stdout. Its intended
+usage is as input to a filter that would take each event one at a time and do
+something with it, test a new revision for instance.
+
+As a quick example of the above, to print the list of series created or updated
+since a specific date, a simple filter can be written:
+
+.. code-block:: python
+
+   #!/bin/python
+   import fileinput
+   import json
+
+   for line in fileinput.input():
+       event = json.loads(line)
+       series = event['series']
+       revision = event['parameters']['revision']
+       print("series %d (rev %d)" % (series, revision))
+
+Which gives::
+
+    $ git pw poll-events --since=2016-02-12 | ./show-series
+    series 3324 (rev 1)
+    series 3304 (rev 3)
+    series 3072 (rev 2)
+    series 3344 (rev 1)
+
+|git pw post-result|
+~~~~~~~~~~~~~~~~~~~~
+
+The other side of the patchwork interaction with testing is sending test
+results back. Here as well |git-pw| provides a command to simplify the process.
+Remember it's always possible to directly use the REST API.
+
+No need to repeat what's written in the
+:http:post:`/test-results/</api/1.0/series/(int: series_id)/revisions/(int: version)/test-results/>` documentation here. Just a couple of examples, setting
+a test result as pending::
+
+    $ git pw post-result 3324 checkpatch.pl pending
+
+And posting the final results::
+
+    $ git pw post-result 3324 checkpatch.pl failure --summary-from-file results.txt
+
+Example: running checkpatch.pl on incoming series
+-------------------------------------------------
+
+A slightly longer example can be found in the Patchwork repository, in
+`docs/examples/testing-checkpatch.py`__. This script will take
+`series-new-revision` events as input, as offered by |git pw poll-events| and
+run checkpatch.pl on the incoming series. The main complexity beyond what has
+been explained in this document is that checkpatch.pl is run on each patch of
+the series individually (by looping on all mails of the series mbox) and the
+checkpatch.pl output is aggregated to be sent in the `summary` field of the
+test result.
+
+Putting the following line in a cron entry should be enough to run
+checkpatch.pl on each new series::
+
+    git pw poll-events | testing-checkpatch.pl
+
+There are a few improvements to make to have a nicer solution: for instance,
+one could make sure that the checkpatch.pl script is up-to-date by updating the
+Linux checkout before running the test.
+
+.. __: https://github.com/dlespiau/patchwork/blob/master/docs/examples/testing-checkpatch.py
+
+.. include:: symbols