[git-webkit] Provide mechanism to exempt bugs from redaction

Tools/Scripts/hooks/pre-push

@@ -81,7 +81,7 @@ try:
     import webkitpy
     from webkitcorepy import Version
     from webkitscmpy import local, Commit
-    from webkitbugspy import version
+    from webkitbugspy import version, Tracker
 
     if MODE != NO_RADAR_MODE and version >= Version(0, 9, 8):
         REPOSITORY = local.Git(os.getcwd())
@@ -234,9 +234,15 @@ def security_level_for_commit(commit, remotes, name_to_remote=None):
     # Class 3: We must inspect the commit for bug references
     if REPOSITORY:
         obj = Commit(hash=commit, message=message)
+        did_redact = False
         for issue in obj.issues:
-            if issue.redacted:
-                return 1, CLASS_3
+            redaction = issue.redacted
+            if redaction:
+                did_redact = True
+            if getattr(redaction, 'exemption', False):
+                return 0, CLASS_3
+        if did_redact:
+            return 1, CLASS_3
         return 0, CLASS_3
 
     return None, None

Tools/Scripts/libraries/webkitbugspy/setup.py

@@ -30,7 +30,7 @@ def readme():
 
 setup(
     name='webkitbugspy',
-    version='0.9.8',
+    version='0.10.0',
     description='Library containing a shared API for various bug trackers.',
     long_description=readme(),
     classifiers=[

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/__init__.py

@@ -46,7 +46,7 @@ def _maybe_add_library_path(path):
         "Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`"
     )
 
-version = Version(0, 9, 8)
+version = Version(0, 10, 0)
 
 from .user import User
 from .issue import Issue

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/bugzilla.py

@@ -61,8 +61,8 @@ def default(context, obj):
                 raise TypeError('Cannot invoke parent class when classmethod')
             return super(Tracker.Encoder, context).default(obj)
 
-    def __init__(self, url, users=None, res=None, login_attempts=3, redact=None, radar_importer=None, hide_title=None):
-        super(Tracker, self).__init__(users=users, redact=redact, hide_title=hide_title)
+    def __init__(self, url, users=None, res=None, login_attempts=3, redact=None, radar_importer=None, hide_title=None, redact_exemption=None):
+        super(Tracker, self).__init__(users=users, redact=redact, redact_exemption=redact_exemption, hide_title=hide_title)
 
         self._logins_left = login_attempts + 1 if login_attempts else 1
         match = self.ROOT_RE.match(url)

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/github.py

@@ -80,8 +80,9 @@ def __init__(
             component_color=DEFAULT_COMPONENT_COLOR,
             version_color=DEFAULT_VERSION_COLOR,
             session=None, redact=None, hide_title=None,
+            redact_exemption=None,
     ):
-        super(Tracker, self).__init__(users=users, redact=redact, hide_title=hide_title)
+        super(Tracker, self).__init__(users=users, redact=redact, hide_title=hide_title, redact_exemption=redact_exemption)
 
         self.session = session or requests.Session()
         self.component_color = component_color

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/issue.py

@@ -207,6 +207,16 @@ def redacted(self):
         match_string = ''
         for member in ('title', 'project', 'component', 'version', 'classification'):
             match_string += ';{}:{}'.format(member, getattr(self, member, ''))
+        match_string += ';keywords:{}'.format(','.join(self.keywords or []))
+
+        for key, value in self.tracker._redact_exemption.items():
+            if key.search(match_string) and value:
+                return self.tracker.Redaction(
+                    redacted=False,
+                    exemption=value,
+                    reason="is a {}".format(self.tracker.NAME) if key.pattern == '.*' else "matches '{}'".format(key.pattern),
+                )
+
         for key, value in self.tracker._redact.items():
             if key.search(match_string):
                 return self.tracker.Redaction(

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/radar.py

@@ -103,9 +103,9 @@ def radarclient():
         except ImportError:
             return None
 
-    def __init__(self, users=None, authentication=None, project=None, projects=None, redact=None, hide_title=None):
+    def __init__(self, users=None, authentication=None, project=None, projects=None, redact=None, hide_title=None, redact_exemption=None):
         hide_title = True if hide_title is None else hide_title
-        super(Tracker, self).__init__(users=users, redact=redact, hide_title=hide_title)
+        super(Tracker, self).__init__(users=users, redact=redact, redact_exemption=redact_exemption, hide_title=hide_title)
         self._projects = [project] if project else (projects or [])
 
         self.library = self.radarclient()

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/tests/bugzilla_unittest.py

@@ -432,6 +432,21 @@ def test_redaction(self):
                 bugzilla.Tracker.Redaction(True, "matches 'version:Other'"),
             )
 
+    def test_redaction_exception(self):
+        with mocks.Bugzilla(self.URL.split('://')[1], issues=mocks.ISSUES, projects=mocks.PROJECTS):
+            self.assertEqual(bugzilla.Tracker(
+                self.URL,
+                redact={'.*': True},
+                redact_exemption={'component:Text': True},
+            ).issue(1).redacted, bugzilla.Tracker.Redaction(
+                exemption=True, reason="matches 'component:Text'",
+            ))
+            self.assertEqual(bugzilla.Tracker(
+                self.URL,
+                redact={'.*': True},
+                redact_exemption={'component:Scrolling': True},
+            ).issue(1).redacted, bugzilla.Tracker.Redaction(True, 'is a Bugzilla'))
+
     def test_cc_no_radar(self):
         with OutputCapture(level=logging.INFO), mocks.Bugzilla(self.URL.split('://')[1], environment=wkmocks.Environment(
             BUGS_EXAMPLE_COM_USERNAME='tcontributor@example.com',

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/tests/github_unittest.py

@@ -398,6 +398,21 @@ def test_redaction(self):
                 github.Tracker.Redaction(True, "matches 'version:Other'"),
             )
 
+    def test_redaction_exception(self):
+        with mocks.GitHub(self.URL.split('://')[1], issues=mocks.ISSUES, projects=mocks.PROJECTS):
+            self.assertEqual(github.Tracker(
+                self.URL,
+                redact={'.*': True},
+                redact_exemption={'component:Text': True},
+            ).issue(1).redacted, github.Tracker.Redaction(
+                exemption=True, reason="matches 'component:Text'",
+            ))
+            self.assertEqual(github.Tracker(
+                self.URL,
+                redact={'.*': True},
+                redact_exemption={'component:Scrolling': True},
+            ).issue(1).redacted, github.Tracker.Redaction(True, 'is a GitHub Issue'))
+
     def test_parse_error(self):
         error_json = {'message': 'Validation Failed', 'errors': [{'resource': 'Issue', 'code': 'custom', 'field': 'body', 'message': 'body is too long (maximum is 65536 characters)'}], 'documentation_url': 'https://docs.github.com/rest/reference/pulls#create-a-pull-request'}
         with mocks.GitHub(self.URL.split('://')[1], issues=mocks.ISSUES):

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/tests/radar_unittest.py

@@ -373,6 +373,21 @@ def test_redaction(self):
                 redact={'version:Other': True},
             ).issue(1).redacted, radar.Tracker.Redaction(True, "matches 'version:Other'"))
 
+    def test_redaction_exception(self):
+        with wkmocks.Environment(RADAR_USERNAME='tcontributor'), mocks.Radar(issues=mocks.ISSUES, projects=mocks.PROJECTS):
+            self.assertEqual(radar.Tracker(
+                project='WebKit',
+                redact={'.*': True},
+                redact_exemption={'component:Text': True},
+            ).issue(1).redacted, radar.Tracker.Redaction(
+                exemption=True, reason="matches 'component:Text'",
+            ))
+            self.assertEqual(radar.Tracker(
+                project='WebKit',
+                redact={'.*': True},
+                redact_exemption={'component:Scrolling': True},
+            ).issue(1).redacted, radar.Tracker.Redaction(True, 'is a Radar'))
+
     def test_milestone(self):
         with mocks.Radar(issues=mocks.ISSUES):
             tracker = radar.Tracker()

Tools/Scripts/libraries/webkitbugspy/webkitbugspy/tracker.py

@@ -44,9 +44,13 @@ def default(self, obj):
             return obj.Encoder.default(obj)
 
     class Redaction(object):
-        def __init__(self, redacted=False, reason=None):
+        def __init__(self, redacted=False, reason=None, exemption=False):
             self.redacted = redacted
             self.reason = reason
+            self.exemption = exemption
+
+            if self.exemption and not self.reason:
+                raise ValueError('Must define a reason for an redact exemption')
 
         def __bool__(self):
             return self.redacted
@@ -55,6 +59,8 @@ def __nonzero__(self):
             return self.redacted
 
         def __repr__(self):
+            if self.exemption:
+                return '{} and is exempt from redaction'.format(self.reason)
             if not self.redacted:
                 return 'is not redacted'
             if self.reason:
@@ -70,7 +76,7 @@ def __eq__(self, other):
             elif isinstance(other, bool):
                 return self.redacted == other
             elif isinstance(other, Tracker.Redaction):
-                return self.redacted == other.redacted and self.reason == other.reason
+                return self.redacted == other.redacted and self.exemption == other.exemption and self.reason == other.reason
             return False
 
         def __ne__(self, other):
@@ -89,6 +95,7 @@ def from_json(cls, data):
 
         unpacked = dict(
             redact=data.get('redact'),
+            redact_exemption=data.get('redact_exemption'),
             hide_title=data.get('hide_title'),
         )
         if data.get('type') in ('bugzilla', 'github'):
@@ -121,19 +128,22 @@ def instance(cls):
             return cls._trackers[0]
         return None
 
-    def __init__(self, users=None, redact=None, hide_title=None):
+    def __init__(self, users=None, redact=None, hide_title=None, redact_exemption=None):
         self.users = users or User.Mapping()
         self.hide_title = False if hide_title is None else hide_title
-        if redact is None:
-            self._redact = {re.compile('.*'): False}
-        elif isinstance(redact, dict):
-            self._redact = {}
-            for key, value in redact.items():
-                if not isinstance(key, string_utils.basestring):
-                    raise ValueError("'{}' is not a string, only strings allowed in redaction mapping".format(key))
-                self._redact[re.compile(key)] = bool(value)
-        else:
-            raise ValueError("Expected redaction mapping to be of type dict, got '{}'".format(type(redact)))
+
+        for name, rvalue in (('redact', redact), ('redact_exemption', redact_exemption)):
+            if rvalue is None:
+                setattr(self, '_{}'.format(name), {re.compile('.*'): False})
+            elif isinstance(rvalue, dict):
+                attribute = {}
+                for key, value in rvalue.items():
+                    if not isinstance(key, string_utils.basestring):
+                        raise ValueError("'{}' is not a string, only strings allowed in redaction mapping".format(key))
+                    attribute[re.compile(key)] = bool(value)
+                setattr(self, '_{}'.format(name), attribute)
+            else:
+                raise ValueError("Expected redaction mapping to be of type dict, got '{}'".format(type(redact)))
 
     @decorators.hybridmethod
     def from_string(context, string):

Tools/Scripts/libraries/webkitscmpy/setup.py

@@ -29,7 +29,7 @@ def readme():
 
 setup(
     name='webkitscmpy',
-    version='6.1.5',
+    version='6.1.6',
     description='Library designed to interact with git and svn repositories.',
     long_description=readme(),
     classifiers=[

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/__init__.py

@@ -46,7 +46,7 @@ def _maybe_add_webkitcorepy_path():
         "Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`"
     )
 
-version = Version(6, 1, 5)
+version = Version(6, 1, 6)
 
 AutoInstall.register(Package('fasteners', Version(0, 15, 0)))
 AutoInstall.register(Package('jinja2', Version(2, 11, 3)))

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/program/pull_request.py

@@ -393,10 +393,21 @@ def create_pull_request(cls, repository, args, branch_point, callback=None, unbl
         radar_cc_default = repository.config().get('webkitscmpy.cc-radar', 'true') == 'true'
         if radar_issue and not_radar and radar_issue.tracker.radarclient() and (args.cc_radar or (radar_cc_default and args.cc_radar is not False)):
             not_radar.cc_radar(radar=radar_issue)
+
+        redaction_exemption = None
         redacted_issue = None
         for candidate in issues:
+            if getattr(candidate.redacted, 'exemption', False):
+                redaction_exemption = candidate
             if candidate.redacted:
                 redacted_issue = candidate
+        if redaction_exemption:
+            print('A commit you are uploading references {}'.format(redaction_exemption.link))
+            print("{} {}".format(redaction_exemption.link, redaction_exemption.redacted))
+            if redacted_issue:
+                sys.stderr.write("Redaction exemption overrides the redaction of {}\n".format(redacted_issue.link))
+                sys.stderr.write("{} {}\n".format(redacted_issue.link, redacted_issue.redacted))
+            redacted_issue = None
         issue = issues[0] if issues else None
 
         remote_repo = repository.remote(name=source_remote)

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/pull_request_unittest.py

@@ -1015,6 +1015,86 @@ def test_github_branch_secondary_redacted(self):
             ],
         )
 
+    def test_github_branch_redaction_exemption(self):
+        with OutputCapture(level=logging.INFO) as captured, mocks.remote.GitHub(projects=bmocks.PROJECTS) as remote, bmocks.Bugzilla(
+            self.BUGZILLA.split('://')[-1],
+            projects=bmocks.PROJECTS, issues=bmocks.ISSUES,
+            environment=Environment(
+                BUGS_EXAMPLE_COM_USERNAME='tcontributor@example.com',
+                BUGS_EXAMPLE_COM_PASSWORD='password',
+            ),
+        ), patch(
+            'webkitbugspy.Tracker._trackers', [bugzilla.Tracker(
+                self.BUGZILLA,
+                redact={'component:Text': True},
+                redact_exemption={'component:Scrolling': True},
+            )]
+        ), mocks.local.Git(
+            self.path, remote='https://{}'.format(remote.remote),
+            remotes=dict(fork='https://{}/Contributor/WebKit'.format(remote.hosts[0])),
+        ) as repo, mocks.local.Svn():
+
+            repo.commits['eng/pr-branch'] = [
+                repo.commits[repo.default_branch][-1],
+                Commit(
+                    hash='06de5d56554e693db72313f4ca1fb969c30b8ccb',
+                    branch='eng/pr-branch',
+                    author=dict(name='Tim Contributor', emails=['tcontributor@example.com']),
+                    identifier="5.1@eng/pr-branch",
+                    timestamp=int(time.time()),
+                    message='[Testing] Existing commit\nbugs.example.com/show_bug.cgi?id=2\nbugs.example.com/show_bug.cgi?id=1\n'
+                )
+            ]
+            repo.head = repo.commits['eng/pr-branch'][-1]
+
+            self.assertEqual(0, program.main(
+                args=('pull-request', '-v', '--no-history'),
+                path=self.path,
+            ))
+
+            self.assertEqual(
+                Tracker.instance().issue(2).comments[-1].content,
+                'Pull request: https://github.example.com/WebKit/WebKit/pull/1',
+            )
+            gh_issue = github.Tracker('https://github.example.com/WebKit/WebKit').issue(1)
+            self.assertEqual(gh_issue.project, 'WebKit')
+            self.assertEqual(gh_issue.component, 'Scrolling')
+
+        self.maxDiff = None
+        self.assertEqual(
+            captured.stdout.getvalue(),
+            "A commit you are uploading references https://bugs.example.com/show_bug.cgi?id=2\n"
+            "https://bugs.example.com/show_bug.cgi?id=2 matches 'component:Scrolling' and is exempt from redaction\n"
+            "Created 'PR 1 | [Testing] Existing commit'!\n"
+            "Posted pull request link to https://bugs.example.com/show_bug.cgi?id=2\n"
+            "https://github.example.com/WebKit/WebKit/pull/1\n",
+        )
+        self.assertEqual(
+            captured.stderr.getvalue(),
+            'Redaction exemption overrides the redaction of https://bugs.example.com/show_bug.cgi?id=1\n'
+            "https://bugs.example.com/show_bug.cgi?id=1 matches 'component:Text' and is thus redacted\n",
+        )
+        log = captured.root.log.getvalue().splitlines()
+        self.assertEqual(
+            [line for line in log if 'Mock process' not in line], [
+                '    Found 1 commit...',
+                'Using committed changes...',
+                "Rebasing 'eng/pr-branch' on 'main'...",
+                "Rebased 'eng/pr-branch' on 'main!'",
+                'Running pre-PR checks...',
+                'No pre-PR checks to run',
+                'Checking if PR already exists...',
+                'PR not found.',
+                "Pushing 'eng/pr-branch' to 'fork'...",
+                "Syncing 'main' to remote 'fork'",
+                "Creating pull-request for 'eng/pr-branch'...",
+                'Checking issue assignee...',
+                'Checking for pull request link in associated issue...',
+                'Syncing PR labels with issue component...',
+                'Synced PR labels with issue component!',
+            ],
+        )
+
     def test_github_reopen_bugzilla(self):
         with OutputCapture(level=logging.INFO) as captured, mocks.remote.GitHub() as remote, bmocks.Bugzilla(
             self.BUGZILLA.split('://')[-1],

metadata/trackers.json

@@ -23,6 +23,10 @@
         "redact" : {
             "classification:Security": true,
             "tentpole:.*Security.*": true
+        },
+        "redact_exemption" : {
+            "keywords:[^:]*WebKit Security Does Not Repro in Shipping" : true,
+            "keywords:[^:]*WebKit Security Merge-Back Approved" : true
         }
     }
 ]