[webkitscmpy] Collect diff from Bitbucket remote repository

https://bugs.webkit.org/show_bug.cgi?id=265705
rdar://119055886

Reviewed by Dewei Zhu.

* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/bitbucket.py:
(BitBucket.commit): 'HEAD' in Bitbucket means "latest commit on the default branch".
(BitBucket.request): Return a response with a mock diff.
* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/bitbucket.py:
(BitBucket.json_to_diff): Convert Bitbucket json diff to text diff.
(BitBucket.commits): Return a list of commits to caller.
(BitBucket.diff): Given a commit or commit range, return a line-by-line diff of
the provided range. If the caller requests it, include the commit messages for
the specified commits in the same patch format used by 'git format-patch'.
* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/git_hub.py:
(GitHub): Move EMAIL_RE to base class.
* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/scm.py:
(Scm): Move EMAIL_RE from GitHub.
* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/git_unittest.py:

Canonical link: https://commits.webkit.org/271688@main

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/remote/bitbucket.py

@@ -84,6 +84,8 @@ def resolve_all_commits(self, branch):
         return all_commits
 
     def commit(self, ref):
+        if ref == 'HEAD':
+            ref = self.default_branch
         if ref in self.commits:
             return self.commits[ref][-1]
         if ref in self.tags:
@@ -217,9 +219,32 @@ def request(self, method, url, data=None, params=None, json=None, **kwargs):
                         )
                     ) for path in ('Source/main.cpp', 'Source/main.h')],
                 ))
-            commit = self.commit(stripped_url.split('/')[-1])
+            commit = self.commit(stripped_url.split('/')[9])
             if not commit:
                 return mocks.Response.create404(url)
+
+            if stripped_url.split('?')[0].endswith('diff'):
+                return mocks.Response.fromJson(dict(
+                    fromHash=None,
+                    toHash=commit.hash,
+                    contextLines=3,
+                    whitespace='SHOW',
+                    diffs=[dict(
+                        source=dict(toString='ChangeLog'),
+                        destination=dict(toString='ChangeLog'),
+                        hunks=[dict(
+                            sourceLine=1,
+                            sourceSpan=0,
+                            destinationLine=1,
+                            destinationSpan=0,
+                            segments=[dict(
+                                type='ADDED',
+                                lines=[dict(line=line) for line in commit.message.splitlines()],
+                            )],
+                        )],
+                    )],
+                ))
+
             return mocks.Response.fromJson(dict(
                 id=commit.hash,
                 displayId=commit.hash[:12],

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/bitbucket.py

@@ -23,6 +23,7 @@
 import re
 import sys
 
+from datetime import datetime
 from webkitcorepy import decorators, string_utils, CallByNeed
 from webkitscmpy import Commit, Contributor, PullRequest
 from webkitscmpy.remote.scm import Scm
@@ -32,6 +33,7 @@
 
 class BitBucket(Scm):
     URL_RE = re.compile(r'\Ahttps?://(?P<domain>\S+)/projects/(?P<project>\S+)/repos/(?P<repository>\S+)\Z')
+    DIFF_CONTEXT = 3
 
     class PRGenerator(Scm.PRGenerator):
         TITLE_CHAR_LIMIT = 254
@@ -347,6 +349,26 @@ def statuses(self, pull_request):
     def is_webserver(cls, url):
         return True if cls.URL_RE.match(url) else False
 
+    @classmethod
+    def json_to_diff(cls, data):
+        output = ''
+        for diff in data.get('diffs', []):
+            output += '--- a/{}\n'.format(diff['source']['toString'])
+            output += '+++ b/{}\n'.format(diff['destination']['toString'])
+            for hunk in diff.get('hunks', []):
+                output += '@@ -{},{} +{},{} @@\n'.format(
+                    hunk['sourceLine'], hunk['sourceSpan'],
+                    hunk['destinationLine'], hunk['destinationSpan'],
+                )
+                for segment in hunk.get('segments', []):
+                    leader = dict(
+                        ADDED='+',
+                        REMOVED='-',
+                    ).get(segment['type'], ' ')
+                    for line in segment.get('lines'):
+                        output += '{}{}\n'.format(leader, line['line'])
+        return output
+
     def __init__(self, url, dev_branches=None, prod_branches=None, contributors=None, id=None, classifier=None):
         match = self.URL_RE.match(url)
         if not match:
@@ -600,6 +622,59 @@ def commit(self, hash=None, revision=None, identifier=None, branch=None, tag=Non
             message=commit_data['message'] if include_log else None,
         )
 
+    def commits(self, begin=None, end=None, include_log=True, include_identifier=True):
+        begin, end = self._commit_range(begin=begin, end=end, include_identifier=include_identifier, include_log=include_log)
+
+        previous = end
+        cached = [previous]
+        while previous:
+            response = self.request('commits/{}~1'.format(previous.hash))
+            if not response:
+                break
+            branch_point = previous.branch_point
+            identifier = previous.identifier
+            if identifier is not None:
+                identifier -= 1
+            if not identifier:
+                identifier = branch_point
+                branch_point = None
+
+            matches = self.GIT_SVN_REVISION.findall(response['message'])
+            revision = int(matches[-1].split('@')[0]) if matches else None
+
+            email_match = self.EMAIL_RE.match(response['author']['emailAddress'])
+
+            previous = Commit(
+                repository_id=self.id,
+                hash=response['id'],
+                revision=revision,
+                branch=end.branch if identifier and branch_point else self.default_branch,
+                identifier=identifier if include_identifier else None,
+                branch_point=branch_point if include_identifier else None,
+                timestamp=int(response['committerTimestamp'] / 1000),
+                author=self.contributors.create(
+                    response['author']['name'],
+                    email_match.group('email') if email_match else None,
+                ), order=0,
+                message=response['message'] if include_log else None,
+            )
+            if not cached or cached[0].timestamp != previous.timestamp:
+                for c in cached:
+                    yield c
+                cached = [previous]
+            else:
+                for c in cached:
+                    c.order += 1
+                cached.append(previous)
+
+            if previous.hash == begin.hash or previous.timestamp < begin.timestamp:
+                previous = None
+                break
+
+        for c in cached:
+            c.order += begin.order
+            yield c
+
     def find(self, argument, include_log=True, include_identifier=True):
         if not isinstance(argument, string_utils.basestring):
             raise ValueError("Expected 'argument' to be a string, not '{}'".format(type(argument)))
@@ -626,6 +701,50 @@ def find(self, argument, include_log=True, include_identifier=True):
             raise ValueError("'{}' is not an argument recognized by git".format(argument))
         return self.commit(hash=commit_data['id'], include_log=include_log, include_identifier=include_identifier)
 
+    def diff(self, head='HEAD', base=None, include_log=False):
+        if base:
+            commits = list(self.commits(dict(argument=base), end=dict(argument=head), include_identifier=False))
+        else:
+            commits = [self.find(head, include_identifier=False), None]
+
+        if not commits or not commits[0]:
+            sys.stderr.write('Failed to find commits required to generate diff\n')
+            return
+        commits = commits[:-1]
+
+        patch_count = 1
+        for commit in commits:
+            response = self.request('commits/{}/diff?contextLines={}'.format(commit.hash, self.DIFF_CONTEXT))
+            if not response:
+                sys.stderr.write('Failed to retrieve diff of {} with status code\n'.format(commit))
+                return
+
+            if include_log and commit.message:
+                yield 'From {}'.format(commit.hash)
+                yield 'From: {} <{}>'.format(commit.author.name, commit.author.email)
+                yield 'Date: {}'.format(datetime.fromtimestamp(commit.timestamp).strftime('%a %b %d %H:%M:%S %Y'))
+                if len(commits) <= 1:
+                    subject = 'Subject: [PATCH]'
+                else:
+                    subject = 'Subject: [PATCH {}/{}]'.format(patch_count, len(commits))
+                for line in commit.message.splitlines():
+                    if subject is None:
+                        yield line
+                    elif line:
+                        subject = '{} {}'.format(subject, line)
+                    else:
+                        yield subject
+                        yield line
+                        subject = None
+                if subject:
+                    yield subject
+                yield '---'
+
+            for line in self.json_to_diff(response).splitlines():
+                yield line
+
+            patch_count += 1
+
     def files_changed(self, argument=None):
         if not argument:
             raise ValueError('No argument provided')

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/git_hub.py

@@ -40,7 +40,6 @@
 
 class GitHub(Scm):
     URL_RE = re.compile(r'\Ahttps?://github.(?P<domain>\S+)/(?P<owner>\S+)/(?P<repository>\S+)\Z')
-    EMAIL_RE = re.compile(r'(?P<email>[^@]+@[^@]+)(@.*)?')
     ACCEPT_HEADER = Tracker.ACCEPT_HEADER
     KNOWN_400_MESSAGES = [
         'No commit found for SHA',

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/remote/scm.py

@@ -27,6 +27,8 @@
 
 
 class Scm(ScmBase):
+    EMAIL_RE = re.compile(r'(?P<email>[^@]+@[^@]+)(@.*)?')
+
     class PRGenerator(object):
         SUPPORTS_DRAFTS = False
 

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/git_unittest.py

@@ -962,3 +962,53 @@ def test_checkout_url(self):
         self.assertEqual(remote.BitBucket(self.remote).checkout_url(ssh=True), 'git@bitbucket.example.com/WEBKIT/webkit.git')
         with self.assertRaises(ValueError):
             remote.BitBucket(self.remote).checkout_url(http=True, ssh=True)
+
+    def test_diff(self):
+        with mocks.remote.BitBucket():
+            repo = remote.BitBucket(self.remote)
+            self.assertEqual([
+                '--- a/ChangeLog',
+                '+++ b/ChangeLog',
+                '@@ -1,0 +1,0 @@',
+                '+Patch Series',
+            ], list(repo.diff(base='bae5d1e90999d4f916a8a15810ccfa43f37a2fd6')))
+
+    def test_diff_with_commit_message(self):
+        with mocks.remote.BitBucket():
+            repo = remote.BitBucket(self.remote)
+            self.assertEqual([
+                'From bae5d1e90999d4f916a8a15810ccfa43f37a2fd6',
+                'From: Jonathan Bedard <jbedard@apple.com>',
+                'Date: {}'.format(datetime.fromtimestamp(1601668000).strftime('%a %b %d %H:%M:%S %Y')),
+                'Subject: [PATCH] 8th commit',
+                '---',
+                '--- a/ChangeLog',
+                '+++ b/ChangeLog',
+                '@@ -1,0 +1,0 @@',
+                '+8th commit',
+            ], list(repo.diff(base='3@main', head='4@main', include_log=True)))
+
+    def test_diff_with_commit_message_multiple(self):
+        self.maxDiff = None
+        with mocks.remote.BitBucket():
+            repo = remote.BitBucket(self.remote)
+            self.assertEqual([
+                'From bae5d1e90999d4f916a8a15810ccfa43f37a2fd6',
+                'From: Jonathan Bedard <jbedard@apple.com>',
+                'Date: {}'.format(datetime.fromtimestamp(1601668000).strftime('%a %b %d %H:%M:%S %Y')),
+                'Subject: [PATCH 1/2] 8th commit',
+                '---',
+                '--- a/ChangeLog',
+                '+++ b/ChangeLog',
+                '@@ -1,0 +1,0 @@',
+                '+8th commit',
+                'From 1abe25b443e985f93b90d830e4a7e3731336af4d',
+                'From: Jonathan Bedard <jbedard@apple.com>',
+                'Date: {}'.format(datetime.fromtimestamp(1601663000).strftime('%a %b %d %H:%M:%S %Y')),
+                'Subject: [PATCH 2/2] 4th commit',
+                '---',
+                '--- a/ChangeLog',
+                '+++ b/ChangeLog',
+                '@@ -1,0 +1,0 @@',
+                '+4th commit',
+            ], list(repo.diff(base='2@main', head='4@main', include_log=True)))