Add support for bazel.exclude-tags which will be used for excluding…

… targets with specific tags.

Summary: - Doing so is especially useful for filtering out targets marked as 'flaky' and those with special requirements.

Test Plan: Unit tests.

Reviewers: naphat

Reviewed By: naphat

Subscribers: changesbot, kylec

Differential Revision: https://tails.corp.dropbox.com/D223628

changes/models/jobplan.py

@@ -197,12 +197,14 @@ def get_build_step_for_job(cls, job_id):
                 logging.error('Project config for project %s is missing `bazel.targets`. job: %s, revision_sha: %s, config: %s', job.project.slug, job.id, job.source.revision_sha, str(project_config), exc_info=True)
                 return jobplan, None
 
+            bazel_exclude_tags = project_config['bazel.exclude-tags']
+
             implementation = LXCBuildStep(
                 cluster=current_app.config['DEFAULT_CLUSTER'],
                 commands=[
                     {'script': get_bazel_setup(), 'type': 'setup'},
-                    {'script': sync_encap_pkgs(project_config), 'type': 'setup'},
-                    {'script': collect_bazel_targets(project_config['bazel.targets']), 'type': 'collect_tests'},
+                    {'script': sync_encap_pkgs(project_config), 'type': 'setup'},  # TODO(anupc): Make this optional
+                    {'script': collect_bazel_targets(project_config['bazel.targets'], bazel_exclude_tags), 'type': 'collect_tests'},
                 ],
                 artifacts=['*.xml'],
                 artifact_search_path=current_app.config['BAZEL_TEST_OUTPUT_RELATIVE_PATH'],

changes/models/project.py

@@ -60,7 +60,8 @@ def get(cls, id):
         return project
 
     _default_config = {
-        'build.file-blacklist': []
+        'build.file-blacklist': [],
+        'bazel.exclude-tags': []
     }
 
     def get_config_path(self):

changes/utils/bazel_setup.py

@@ -30,7 +30,9 @@
 (sudo apt-get -y update || sudo apt-get -y update) >/dev/null 2>&1
 sudo apt-get install -y --force-yes %(bazel_apt_pkgs)s python >/dev/null 2>&1
 
-(/usr/bin/bazel --nomaster_blazerc --blazerc=/dev/null --batch query 'tests(%(bazel_targets)s)' | python -c "%(jsonify_script)s") 2> /dev/null
+(/usr/bin/bazel --nomaster_blazerc --blazerc=/dev/null --batch query \
+    'let t = tests(%(bazel_targets)s) in ($t %(exclusion_subquery)s)' | \
+    python -c "%(jsonify_script)s") 2> /dev/null
 """.strip()
 
 
@@ -46,15 +48,50 @@ def get_bazel_setup():
     )
 
 
-def collect_bazel_targets(bazel_targets):
+def collect_bazel_targets(bazel_targets, bazel_exclude_tags):
+    """Construct a command to query the Bazel dependency graph to expand bazel project
+    config into a set of individual test targets.
+
+    Bazel project config currently supports the following attributes:
+    - targets: List of Bazel target patterns (conforming to the spec given in
+      https://www.bazel.io/docs/bazel-user-manual.html#target-patterns). These patterns
+      are additive, meaning a union of all targets matching any of the patterns will be
+      returned.
+    - exclude-tags: List of target tags. Targets matching any of these tags are not returned.
+      By default, this list is empty.  # TODO(anupc): Should we remove `manual` test targets?
+    """
     package_dir = os.path.dirname(__file__)
     bazel_target_py = os.path.join(package_dir, "collect_bazel_targets.py")
+
+    # Please use https://www.bazel.io/docs/query.html as a reference for Bazel query syntax
+    #
+    # To exclude targets matching a tag, we construct a query as follows:
+    #   let t = test(bazel_targets)                     ## Collect list of test targets in $t
+    #   return $t except attr("tags", $expression, $t)  ## Return all targets in t except those where "tags" matches an expression
+    #
+    # Multiple exclusions a performed by adding further "except" clauses
+    #   return $t except attr(1) expect attr(2) except attr(3) ...
+    #
+    # Examples of "tags" attribute:
+    #   []                                              ## No tags
+    #   [flaky]                                         ## Single tag named flaky
+    #   [flaky, manual]                                 ## Two tags named flaky and manual
+    #
+    # Tags are delimited on the left by an opening bracket or space, and on the right by a comma or closing bracket.
+    #
+    # Hence, $expression =>
+    #   (\[| )                                          ## Starts with an opening bracket or space
+    #   tag_name                                        ## Match actual tag name
+    #   (\]|,)                                          ## Ends with a closing bracket or comma
+    exclusion_subquery = ' '.join(["""except (attr("tags", "(\[| )%s(\]|,)", $t))""" % (tag) for tag in bazel_exclude_tags])
+
     with open(bazel_target_py, 'r') as jsonify_script:
         return COLLECT_BAZEL_TARGETS % dict(
             apt_spec=current_app.config['APT_SPEC'],
             bazel_apt_pkgs=' '.join(current_app.config['BAZEL_APT_PKGS']),
             bazel_targets=' + '.join(bazel_targets),
-            jsonify_script=jsonify_script.read()
+            jsonify_script=jsonify_script.read(),
+            exclusion_subquery=exclusion_subquery,
         )
 
 

tests/changes/models/test_job.py

@@ -24,7 +24,8 @@ def test_autogenerated_commands(self, get_config):
                     'package1',
                     'pkg-2',
                 ]
-            }
+            },
+            'bazel.exclude-tags': [],  # Default, specified in models/project.py
         }
 
         current_app.config['APT_SPEC'] = 'deb http://example.com/debian distribution component1'
@@ -73,7 +74,9 @@ def test_autogenerated_commands(self, get_config):
 (sudo apt-get -y update || sudo apt-get -y update) >/dev/null 2>&1
 sudo apt-get install -y --force-yes bazel python >/dev/null 2>&1
 
-(/usr/bin/bazel --nomaster_blazerc --blazerc=/dev/null --batch query 'tests(//aa/bb/cc/... + //aa/abc/...)' | python -c "import sys
+(/usr/bin/bazel --nomaster_blazerc --blazerc=/dev/null --batch query \
+    'let t = tests(//aa/bb/cc/... + //aa/abc/...) in ($t )' | \
+    python -c "import sys
 import json
 
 
@@ -99,3 +102,64 @@ def test_autogenerated_commands(self, get_config):
 
         assert implementation.artifact_search_path == os.path.join(DEFAULT_PATH, current_app.config['BAZEL_TEST_OUTPUT_RELATIVE_PATH'])
         assert implementation.artifacts == ['*.xml']
+
+    @mock.patch('changes.models.project.Project.get_config')
+    def test_autogenerated_commands_with_exclusions(self, get_config):
+        get_config.return_value = {
+            'bazel.targets': [
+                '//foo/bar/baz/...',
+                '//bar/bax/...',
+            ],
+            'bazel.exclude-tags': [
+                'flaky',
+                'another_tag',
+            ],
+        }
+
+        current_app.config['APT_SPEC'] = 'deb http://example.com/debian distribution component1'
+        current_app.config['BAZEL_APT_PKGS'] = ['bazel']
+
+        project = self.create_project()
+        plan = self.create_plan(project)
+        option = self.create_option(
+            item_id=plan.id,
+            name='bazel.autogenerate',
+            value='1',
+        )
+        build = self.create_build(project)
+        job = self.create_job(build)
+        jobplan = self.create_job_plan(job, plan)
+
+        _, implementation = JobPlan.get_build_step_for_job(job.id)
+
+        collect_tests_expected = """#!/bin/bash -eu
+# Clean up any existing apt sources
+sudo rm -rf /etc/apt/sources.list.d >/dev/null 2>&1
+# Overwrite apt sources
+(echo "deb http://example.com/debian distribution component1" | sudo tee /etc/apt/sources.list) >/dev/null 2>&1
+
+# apt-get update, and try again if it fails first time
+(sudo apt-get -y update || sudo apt-get -y update) >/dev/null 2>&1
+sudo apt-get install -y --force-yes bazel python >/dev/null 2>&1
+
+(/usr/bin/bazel --nomaster_blazerc --blazerc=/dev/null --batch query \
+    'let t = tests(//foo/bar/baz/... + //bar/bax/...) in ($t except (attr("tags", "(\[| )flaky(\]|,)", $t)) except (attr("tags", "(\[| )another_tag(\]|,)", $t)))' | \
+    python -c "import sys
+import json
+
+
+targets = sys.stdin.read().splitlines()
+out = {
+    'cmd': '/usr/bin/bazel test {test_names}',
+    'tests': targets,
+}
+json.dump(out, sys.stdout)
+") 2> /dev/null
+""".strip()
+
+        assert len(implementation.commands) == 3
+
+        assert implementation.commands[0].type == CommandType.setup
+        assert implementation.commands[1].type == CommandType.setup
+        assert implementation.commands[2].type == CommandType.collect_tests
+        assert implementation.commands[2].script == collect_tests_expected