have changes pass vcs commands to changes-client during targets colle…

…ction

Summary:
The following commands are passed as env variables:
- checkout current revision
- checkout parent revision
- get the list of changed files from current revision

Test Plan: unit tests

Reviewers: anupc

Reviewed By: anupc

Subscribers: wwu, changesbot, kylec

Differential Revision: https://tails.corp.dropbox.com/D228841

changes/models/jobplan.py

@@ -236,6 +236,7 @@ def get_build_step_for_job(cls, job_id):
                         bazel_exclude_tags=bazel_exclude_tags,
                         max_jobs=2 * bazel_cpus,
                         bazel_test_flags=bazel_test_flags,
+                        vcs=job.project.repository.get_vcs(),
                         ), 'type': 'collect_bazel_targets'},
                 ],
                 artifacts=[],  # only for collect_target step, which we don't expect artifacts

changes/utils/bazel_setup.py

@@ -28,6 +28,10 @@
 (sudo apt-get -y update || sudo apt-get -y update) >/dev/null 2>&1
 sudo apt-get install -y --force-yes {bazel_apt_pkgs} python >/dev/null 2>&1
 
+VCS_CHECKOUT_TARGET_REVISION="{checkout_target_revision}"
+VCS_CHECKOUT_PARENT_REVISION="{checkout_parent_revision}"
+VCS_GET_CHANGED_FILES="{get_changed_files}"
+
 "{collect_targets_executable}" --output-user-root="{bazel_root}" {bazel_targets} {bazel_exclude_tags} {bazel_test_flags} --jobs="{max_jobs}" 2> /dev/null
 """.strip()
 
@@ -45,7 +49,7 @@ def get_bazel_setup():
     )
 
 
-def collect_bazel_targets(collect_targets_executable, bazel_targets, bazel_exclude_tags, bazel_test_flags, max_jobs):
+def collect_bazel_targets(collect_targets_executable, bazel_targets, bazel_exclude_tags, bazel_test_flags, max_jobs, vcs):
     """Construct a command to query the Bazel dependency graph to expand bazel project
     config into a set of individual test targets.
 
@@ -57,7 +61,7 @@ def collect_bazel_targets(collect_targets_executable, bazel_targets, bazel_exclu
     - exclude-tags: List of target tags. Targets matching any of these tags are not returned.
       By default, this list is empty.
     """
-    # type: (str, List[str], List[str], int) -> str
+    # type: (str, List[str], List[str], int, changes.vcs.base.Vcs) -> str
     return COLLECT_BAZEL_TARGETS.format(
         apt_spec=current_app.config['APT_SPEC'],
         bazel_apt_pkgs=' '.join(current_app.config['BAZEL_APT_PKGS']),
@@ -67,6 +71,9 @@ def collect_bazel_targets(collect_targets_executable, bazel_targets, bazel_exclu
         bazel_exclude_tags=' '.join(['--exclude-tags={}'.format(t) for t in bazel_exclude_tags]),
         bazel_test_flags=' '.join(['--test-flags={}'.format(tf) for tf in bazel_test_flags]),
         max_jobs=max_jobs,
+        checkout_target_revision=vcs.get_buildstep_checkout_revision('master'),
+        checkout_parent_revision=vcs.get_buildstep_checkout_parent_revision('master'),
+        get_changed_files=vcs.get_buildstep_changed_files('master'),
     )
 
 

changes/vcs/base.py

@@ -235,6 +235,18 @@ def get_buildstep_clone(self, source, workspace, clean=True, cache_dir="/dev/nul
     def get_buildstep_patch(self, source, workspace):
         raise NotImplementedError
 
+    def get_buildstep_checkout_revision(self, revision_sha):
+        # type: (str) -> str
+        raise NotImplementedError
+
+    def get_buildstep_checkout_parent_revision(self, revision_sha):
+        # type: (str) -> str
+        raise NotImplementedError
+
+    def get_buildstep_changed_files(self, revision_sha):
+        # type: (str) -> str
+        raise NotImplementedError
+
     def log_timing(self, command, start_time):
         repo_type = 'unknown'
         classname = self.__class__.__name__

changes/vcs/git.py

@@ -338,6 +338,18 @@ def get_buildstep_patch(self, source, workspace):
             patch_url=build_patch_uri(source.patch_id),
         )
 
+    def get_buildstep_checkout_revision(self, revision_sha):
+        # type: (str) -> str
+        return "#!/bin/bash\n/usr/bin/git checkout {sha}".format(sha=revision_sha)
+
+    def get_buildstep_checkout_parent_revision(self, revision_sha):
+        # type: (str) -> str
+        return "#!/bin/bash\n/usr/bin/git checkout {sha}^".format(sha=revision_sha)
+
+    def get_buildstep_changed_files(self, revision_sha):
+        # type: (str) -> str
+        return "#!/bin/bash\n/usr/bin/git diff --name-only {sha}^..{sha}".format(sha=revision_sha)
+
     def read_file(self, sha, file_path, diff=None):
         """Read the content of a file at a given revision.
 

tests/changes/models/test_job.py

@@ -7,6 +7,7 @@
 from changes.models.command import CommandType
 from changes.models.jobplan import JobPlan
 from changes.testutils import TestCase
+from changes.vcs.base import Vcs
 
 
 class AutogeneratedJobTest(TestCase):
@@ -47,7 +48,14 @@ def test_autogenerated_commands(self, get_config):
             'bazel.max-executors': 1,  # Default
         }
 
-        _, implementation = JobPlan.get_build_step_for_job(self._create_job_and_jobplan().id)
+        mock_vcs = mock.Mock(spec=Vcs)
+        mock_vcs.get_buildstep_checkout_revision.return_value = 'git checkout master'
+        mock_vcs.get_buildstep_checkout_parent_revision.return_value = 'git checkout master^'
+        mock_vcs.get_buildstep_changed_files.return_value = 'git diff --name-only master^..master'
+        job = self._create_job_and_jobplan()
+        with mock.patch.object(job.project.repository, "get_vcs") as mock_get_vcs:
+            mock_get_vcs.return_value = mock_vcs
+            _, implementation = JobPlan.get_build_step_for_job(job.id)
 
         bazel_setup_expected = """#!/bin/bash -eux
 # Clean up any existing apt sources
@@ -78,6 +86,10 @@ def test_autogenerated_commands(self, get_config):
 (sudo apt-get -y update || sudo apt-get -y update) >/dev/null 2>&1
 sudo apt-get install -y --force-yes bazel python >/dev/null 2>&1
 
+VCS_CHECKOUT_TARGET_REVISION="git checkout master"
+VCS_CHECKOUT_PARENT_REVISION="git checkout master^"
+VCS_GET_CHANGED_FILES="git diff --name-only master^..master"
+
 "/var/changes/input/collect-targets" --output-user-root="/bazel/root/path" --target-patterns=//aa/bb/cc/... --target-patterns=//aa/abc/...  --test-flags=--spawn_strategy=sandboxed --test-flags=--genrule_strategy=sandboxed --jobs="8" 2> /dev/null
 """.strip()
 
@@ -123,10 +135,21 @@ def test_autogenerated_commands_with_exclusions(self, get_config):
 (sudo apt-get -y update || sudo apt-get -y update) >/dev/null 2>&1
 sudo apt-get install -y --force-yes bazel python >/dev/null 2>&1
 
+VCS_CHECKOUT_TARGET_REVISION="git checkout master"
+VCS_CHECKOUT_PARENT_REVISION="git checkout master^"
+VCS_GET_CHANGED_FILES="git diff --name-only master^..master"
+
 "/var/changes/input/collect-targets" --output-user-root="/bazel/root/path" --target-patterns=//foo/bar/baz/... --target-patterns=//bar/bax/... --exclude-tags=flaky --exclude-tags=another_tag --test-flags=--spawn_strategy=sandboxed --test-flags=--genrule_strategy=sandboxed --jobs="4" 2> /dev/null
 """.strip()
 
-        _, implementation = JobPlan.get_build_step_for_job(self._create_job_and_jobplan().id)
+        mock_vcs = mock.Mock(spec=Vcs)
+        mock_vcs.get_buildstep_checkout_revision.return_value = 'git checkout master'
+        mock_vcs.get_buildstep_checkout_parent_revision.return_value = 'git checkout master^'
+        mock_vcs.get_buildstep_changed_files.return_value = 'git diff --name-only master^..master'
+        job = self._create_job_and_jobplan()
+        with mock.patch.object(job.project.repository, "get_vcs") as mock_get_vcs:
+            mock_get_vcs.return_value = mock_vcs
+            _, implementation = JobPlan.get_build_step_for_job(job.id)
 
         assert implementation.max_executors == 3