gitlab hook: support merge_request events

common/code_spelling_ignore_words.txt

@@ -556,6 +556,7 @@ gf
 gib
 giger
 github
+gitlab
 gitorious
 gmail
 gmt

master/buildbot/newsfragments/gitlabhook.feature

@@ -0,0 +1,2 @@
+Gitlab hook now supports the merge_request event to automatically build from a merge request.
+Note that the results will not properly displayed in merge_request UI due to https://gitlab.com/gitlab-org/gitlab-ce/issues/33293

master/buildbot/test/unit/test_www_hooks_gitlab.py

@@ -30,7 +30,7 @@
 from buildbot.www.hooks.gitlab import _HEADER_GITLAB_TOKEN
 
 
-# Sample GITHUB commit payload from http://help.github.com/post-receive-hooks/
+# Sample GITLAB commit payload from https://docs.gitlab.com/ce/user/project/integrations/webhooks.html
 # Added "modified" and "removed", and change email
 gitJsonPayload = """
 {
@@ -113,6 +113,85 @@
    "total_commits_count": 2
 }
 """
+gitJsonPayloadMR = """
+{
+  "object_kind": "merge_request",
+  "user": {
+    "name": "Administrator",
+    "username": "root",
+    "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon"
+  },
+  "object_attributes": {
+    "id": 99,
+    "target_branch": "master",
+    "source_branch": "ms-viewport",
+    "source_project_id": 14,
+    "author_id": 51,
+    "assignee_id": 6,
+    "title": "MS-Viewport",
+    "created_at": "2013-12-03T17:23:34Z",
+    "updated_at": "2013-12-03T17:23:34Z",
+    "st_commits": null,
+    "st_diffs": null,
+    "milestone_id": null,
+    "state": "opened",
+    "merge_status": "unchecked",
+    "target_project_id": 14,
+    "iid": 1,
+    "description": "",
+    "source":{
+      "name":"Awesome Project",
+      "description":"Aut reprehenderit ut est.",
+      "web_url":"http://example.com/awesome_space/awesome_project",
+      "avatar_url":null,
+      "git_ssh_url":"git@example.com:awesome_space/awesome_project.git",
+      "git_http_url":"http://example.com/awesome_space/awesome_project.git",
+      "namespace":"Awesome Space",
+      "visibility_level":20,
+      "path_with_namespace":"awesome_space/awesome_project",
+      "default_branch":"master",
+      "homepage":"http://example.com/awesome_space/awesome_project",
+      "url":"http://example.com/awesome_space/awesome_project.git",
+      "ssh_url":"git@example.com:awesome_space/awesome_project.git",
+      "http_url":"http://example.com/awesome_space/awesome_project.git"
+    },
+    "target": {
+      "name":"Awesome Project",
+      "description":"Aut reprehenderit ut est.",
+      "web_url":"http://example.com/awesome_space/awesome_project",
+      "avatar_url":null,
+      "git_ssh_url":"git@example.com:awesome_space/awesome_project.git",
+      "git_http_url":"http://example.com/awesome_space/awesome_project.git",
+      "namespace":"Awesome Space",
+      "visibility_level":20,
+      "path_with_namespace":"awesome_space/awesome_project",
+      "default_branch":"master",
+      "homepage":"http://example.com/awesome_space/awesome_project",
+      "url":"http://example.com/awesome_space/awesome_project.git",
+      "ssh_url":"git@example.com:awesome_space/awesome_project.git",
+      "http_url":"http://example.com/awesome_space/awesome_project.git"
+    },
+    "last_commit": {
+      "id": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+      "message": "fixed readme",
+      "timestamp": "2012-01-03T23:36:29+02:00",
+      "url": "http://example.com/awesome_space/awesome_project/commits/da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+      "author": {
+        "name": "GitLab dev user",
+        "email": "gitlabdev@dv6700.(none)"
+      }
+    },
+    "work_in_progress": false,
+    "url": "http://example.com/diaspora/merge_requests/1",
+    "action": "open",
+    "assignee": {
+      "name": "User1",
+      "username": "user1",
+      "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon"
+    }
+  }
+}
+"""
 
 
 class TestChangeHookConfiguredWithGitChange(unittest.TestCase):
@@ -132,6 +211,18 @@ def check_changes_tag_event(self, r, project='', codebase=None):
         )
         self.assertEqual(change["branch"], "v1.0.0")
 
+    def check_changes_mr_event(self, r, project='', codebase=None):
+        self.assertEqual(len(self.changeHook.master.addedChanges), 1)
+        change = self.changeHook.master.addedChanges[0]
+
+        self.assertEqual(change["repository"], "http://example.com/awesome_space/awesome_project.git")
+        self.assertEqual(
+            calendar.timegm(change["when_timestamp"].utctimetuple()),
+            1325626589
+        )
+        self.assertEqual(change["branch"], "refs/merge-requests/1/head")
+        self.assertEqual(change["category"], "merge_request")
+
     def check_changes_push_event(self, r, project='', codebase=None):
         self.assertEqual(len(self.changeHook.master.addedChanges), 2)
         change = self.changeHook.master.addedChanges[0]
@@ -177,6 +268,7 @@ def testGitWithChange(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.uri = b"/change_hook/gitlab"
         self.request.method = b"POST"
+        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_push_event(res)
 
@@ -185,6 +277,7 @@ def testGitWithChange_WithProjectToo(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'project': ['MyProject']}
+        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
         self.request.method = b"POST"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_push_event(res, project="MyProject")
@@ -194,6 +287,7 @@ def testGitWithChange_WithCodebaseToo(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'codebase': ['MyCodebase']}
+        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
         self.request.method = b"POST"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_push_event(res, codebase="MyCodebase")
@@ -203,6 +297,7 @@ def testGitWithChange_WithPushTag(self):
         self.request = FakeRequest(content=gitJsonPayloadTag)
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'codebase': ['MyCodebase']}
+        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
         self.request.method = b"POST"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_tag_event(res, codebase="MyCodebase")
@@ -211,6 +306,7 @@ def testGitWithNoJson(self):
         self.request = FakeRequest()
         self.request.uri = b"/change_hook/gitlab"
         self.request.method = b"POST"
+        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
         d = self.request.test_render(self.changeHook)
 
         def check_changes(r):
@@ -231,6 +327,19 @@ def test_event_property(self):
         self.assertEqual(len(self.changeHook.master.addedChanges), 2)
         change = self.changeHook.master.addedChanges[0]
         self.assertEqual(change["properties"]["event"], "Push Hook")
+        self.assertEqual(change["category"], "Push Hook")
+
+    @defer.inlineCallbacks
+    def testGitWithChange_WithMR(self):
+        self.request = FakeRequest(content=gitJsonPayloadMR)
+        self.request.uri = b"/change_hook/gitlab"
+        self.request.args = {'codebase': ['MyCodebase']}
+        self.request.received_headers[_HEADER_EVENT] = "Merge Request Hook"
+        self.request.method = b"POST"
+        res = yield self.request.test_render(self.changeHook)
+        self.check_changes_mr_event(res, codebase="MyCodebase")
+        change = self.changeHook.master.addedChanges[0]
+        self.assertEqual(change["category"], "merge_request")
 
 
 class TestChangeHookConfiguredWithSecret(unittest.TestCase):
@@ -248,6 +357,7 @@ def test_missing_secret(self):
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'codebase': ['MyCodebase']}
         self.request.method = b"POST"
+        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
         yield self.request.test_render(self.changeHook)
         expected = b'Invalid secret'
         self.assertEqual(self.request.written, expected)
@@ -257,6 +367,7 @@ def test_missing_secret(self):
     def test_valid_secret(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.received_headers[_HEADER_GITLAB_TOKEN] = self._SECRET
+        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
         self.request.uri = b"/change_hook/gitlab"
         self.request.method = b"POST"
         yield self.request.test_render(self.changeHook)

master/buildbot/www/hooks/gitlab.py

@@ -36,7 +36,7 @@ def _process_change(payload, user, repo, repo_url, project, event,
 
     :arguments:
         payload
-            Python Object that represents the JSON sent by GitHub Service
+            Python Object that represents the JSON sent by GitLab Service
             Hook.
     """
     changes = []
@@ -78,6 +78,7 @@ def _process_change(payload, user, repo, repo_url, project, event,
             'revlink': commit['url'],
             'repository': repo_url,
             'project': project,
+            'category': event,
             'properties': {
                 'event': event,
             },
@@ -91,6 +92,42 @@ def _process_change(payload, user, repo, repo_url, project, event,
     return changes
 
 
+def _process_merge_request_change(payload, project, event, codebase=None):
+    """
+    Consumes the merge_request JSON as a python object and turn it into a buildbot change.
+
+    :arguments:
+        payload
+            Python Object that represents the JSON sent by GitLab Service
+            Hook.
+    """
+    attrs = payload['object_attributes']
+    commit = attrs['last_commit']
+    when_timestamp = dateparse(commit['timestamp'])
+    # @todo provide and document a way to choose between http and ssh url
+    repo_url = attrs['target']['git_http_url']
+    changes = [{
+        'author': '%s <%s>' % (commit['author']['name'],
+                               commit['author']['email']),
+        'files': [],  # @todo use rest API
+        'comments': "MR#{}: {}\n\n{}".format(attrs['iid'], attrs['title'], attrs['description']),
+        'revision': commit['id'],
+        'when_timestamp': when_timestamp,
+        'branch': "refs/merge-requests/{}/head".format(attrs['iid']),
+        'repository': repo_url.replace(":30302", ''),
+        'project': project,
+        'category': event,
+        'revlink': attrs['url'],
+        'properties': {
+            'target_branch': attrs['target_branch'],
+            'event': event,
+        },
+    }]
+    if codebase is not None:
+        changes[0]['codebase'] = codebase
+    return changes
+
+
 def getChanges(request, options=None):
     """
     Reponds only to POST events and starts the build process
@@ -110,16 +147,23 @@ def getChanges(request, options=None):
         raise ValueError("Error loading JSON: " + str(e))
     event_type = request.getHeader(_HEADER_EVENT)
     event_type = bytes2NativeString(event_type)
-    user = payload['user_name']
-    repo = payload['repository']['name']
-    repo_url = payload['repository']['url']
+    # newer version of gitlab have a object_kind parameter,
+    # which allows not to use the http header
+    event_type = payload.get('object_kind', event_type)
     project = request.args.get('project', [''])[0]
-    codebase = request.args.get('codebase', None)
-    if codebase:
-        codebase = codebase[0]
-    # This field is unused:
-    # private = payload['repository']['private']
-    changes = _process_change(
-        payload, user, repo, repo_url, project, event_type, codebase=codebase)
-    log.msg("Received %s changes from gitlab" % len(changes))
+    codebase = request.args.get('codebase', [None])[0]
+    if event_type in ("push", "tag_push", "Push Hook"):
+        user = payload['user_name']
+        repo = payload['repository']['name']
+        repo_url = payload['repository']['url']
+        changes = _process_change(
+            payload, user, repo, repo_url, project, event_type, codebase=codebase)
+    elif event_type == 'merge_request':
+        changes = _process_merge_request_change(
+            payload, project, event_type, codebase=codebase)
+    else:
+        changes = []
+    if changes:
+        log.msg("Received {} changes from {} gitlab event".format(
+            len(changes), event_type))
     return (changes, 'git')