Permalink
Browse files

Implement Bazel culprit finder for Bazel downstream jobs (#356)

Change-Id: Iac9a39a5cd51747ec667554aaa562167c27fc33c
  • Loading branch information...
meteorcloudy committed Oct 18, 2018
1 parent d352b6d commit 20d45609ea47065781ce44c363cdfca4441e4d30
Showing with 231 additions and 7 deletions.
  1. +26 −0 buildkite/README.md
  2. +34 −7 buildkite/bazelci.py
  3. +171 −0 buildkite/culprit_finder.py
View
@@ -89,3 +89,29 @@ for an example.
[buildkite verify pull request]: https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/docs/assets/buildkite-verify-pull-request.png
[pull request details]: https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/docs/assets/pull-request-details.png
[buildkite useful buttons]: https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/docs/assets/buildkite-useful-buttons.png
### Culprit Finder
[Bazel downstream projects](https://buildkite.com/bazel/bazel-with-downstream-projects-bazel) is red? Use culprit finder to find out which bazel commit broke it!
First you should check if the project is green with the latest Bazel release. If not, probably it's their commits that broke the CI.
If a project is green with release Bazel but red with Bazel nightly, it means some Bazel commit broke it, then culprit finder can help!
Create "New Build" in the [Culprit Finder](https://buildkite.com/bazel/culprit-finder) project with the following environment variable:
- PROJECT_NAME (The project name must exists in DOWNSTREAM_PROJECTS in [bazelci.py](https://github.com/bazelbuild/continuous-integration/blob/master/buildkite/bazelci.py))
- PLATFORM_NAME (The platform name must exists in PLATFORMS in [bazelci.py](https://github.com/bazelbuild/continuous-integration/blob/master/buildkite/bazelci.py))
- GOOD_BAZEL_COMMIT (A full Bazel commit, Bazel built at this commit still works for this project)
- BAD_BAZEL_COMMIT (A full Bazel commit, Bazel built at this commit fails with this project)
eg.
```
PROJECT_NAME=rules_go
PLATFORM_NAME=ubuntu1404
GOOD_BAZEL_COMMIT=b6ea3b6caa7f379778e74da33d1bd0ff6477f963
BAD_BAZEL_COMMIT=91eb3d207714af0ab1e5812252a0f10f40d6e4a8
```
Note: Bazel commit can only be set to commits after [63453bdbc6b05bd201375ee9e25b35010ae88aab](https://github.com/bazelbuild/bazel/commit/63453bdbc6b05bd201375ee9e25b35010ae88aab), Culprit Finder needs to download Bazel at specific commit, but we didn't prebuilt Bazel binaries before this commit.
View
@@ -382,8 +382,8 @@ def print_expanded_group(name):
eprint("\n\n+++ {0}\n\n".format(name))
def execute_commands(config, platform, git_repository, use_but, save_but,
build_only, test_only, monitor_flaky_tests):
def execute_commands(config, platform, git_repository, git_repo_location, use_bazel_at_commit,
use_but, save_but, build_only, test_only, monitor_flaky_tests):
fail_pipeline = False
tmpdir = None
bazel_binary = "bazel"
@@ -393,17 +393,27 @@ def execute_commands(config, platform, git_repository, use_but, save_but,
if build_only and test_only:
raise BuildkiteException("build_only and test_only cannot be true at the same time")
if use_bazel_at_commit and use_but:
raise BuildkiteException("use_bazel_at_commit cannot be set when use_but is true")
tmpdir = tempfile.mkdtemp()
sc_process = None
try:
if git_repository:
clone_git_repository(git_repository, platform)
if git_repo_location:
os.chdir(git_repo_location)
else:
clone_git_repository(git_repository, platform)
else:
git_repository = os.getenv("BUILDKITE_REPO")
if is_pull_request() and not is_trusted_author(github_user_for_pull_request(), git_repository):
update_pull_request_verification_status(git_repository, commit, state="success")
if use_bazel_at_commit:
print_collapsed_group(":gcloud: Downloading Bazel built at " + use_bazel_at_commit)
bazel_binary = download_bazel_binary_at_commit(tmpdir, platform, use_bazel_at_commit)
if use_but:
print_collapsed_group(":gcloud: Downloading Bazel Under Test")
bazel_binary = download_bazel_binary(tmpdir, platform)
@@ -671,6 +681,18 @@ def download_bazel_binary(dest_dir, platform):
return bazel_binary_path
def download_bazel_binary_at_commit(dest_dir, platform, bazel_git_commit):
# We only build bazel binary on ubuntu14.04 for every bazel commit.
# It should be OK to use it on other ubuntu platforms.
if "ubuntu" in PLATFORMS[platform].get("host-platform", platform):
platform = "ubuntu1404"
bazel_binary_path = os.path.join(dest_dir, "bazel.exe" if platform == "windows" else "bazel")
execute_command([gsutil_command(), "cp", bazelci_builds_gs_url(platform, bazel_git_commit),
bazel_binary_path])
st = os.stat(bazel_binary_path)
os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
return bazel_binary_path
def clone_git_repository(git_repository, platform):
root = downstream_projects_root(platform)
project_name = re.search(r"/([^/]+)\.git$", git_repository).group(1)
@@ -702,6 +724,7 @@ def clone_git_repository(git_repository, platform):
execute_command(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"])
execute_command(["git", "clean", "-fdqx"])
execute_command(["git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx"])
return clone_path
def execute_batch_commands(commands):
@@ -1191,7 +1214,7 @@ def bazelci_builds_download_url(platform, git_commit):
return "https://storage.googleapis.com/bazel-builds/artifacts/{0}/{1}/bazel".format(platform, git_commit)
def bazelci_builds_upload_url(platform, git_commit):
def bazelci_builds_gs_url(platform, git_commit):
return "gs://bazel-builds/artifacts/{0}/{1}/bazel".format(platform, git_commit)
@@ -1250,7 +1273,7 @@ def try_publish_binaries(build_number, expected_generation):
try:
bazel_binary_path = download_bazel_binary(tmpdir, platform)
execute_command([gsutil_command(), "cp", "-a", "public-read", bazel_binary_path,
bazelci_builds_upload_url(platform, git_commit)])
bazelci_builds_gs_url(platform, git_commit)])
info["platforms"][platform] = {
"url": bazelci_builds_download_url(platform, git_commit),
"sha256": sha256_hexdigest(bazel_binary_path),
@@ -1310,7 +1333,7 @@ def publish_binaries():
def main(argv=None):
if argv is None:
argv = sys.argv
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description="Bazel Continuous Integration Script")
@@ -1339,6 +1362,8 @@ def main(argv=None):
runner.add_argument("--file_config", type=str)
runner.add_argument("--http_config", type=str)
runner.add_argument("--git_repository", type=str)
runner.add_argument("--git_repo_location", type=str, help="Use an existing repository instead of cloning from github")
runner.add_argument("--use_bazel_at_commit", type=str, help="Use Bazel binariy built at a specifc commit")
runner.add_argument("--use_but", type=bool, nargs="?", const=True)
runner.add_argument("--save_but", type=bool, nargs="?", const=True)
runner.add_argument("--build_only", type=bool, nargs="?", const=True)
@@ -1347,7 +1372,7 @@ def main(argv=None):
runner = subparsers.add_parser("publish_binaries")
args = parser.parse_args()
args = parser.parse_args(argv)
try:
if args.subparsers_name == "bazel_publish_binaries_pipeline":
@@ -1374,6 +1399,8 @@ def main(argv=None):
execute_commands(config=configs.get("platforms", None)[args.platform],
platform=args.platform,
git_repository=args.git_repository,
git_repo_location=args.git_repo_location,
use_bazel_at_commit=args.use_bazel_at_commit,
use_but=args.use_but,
save_but=args.save_but,
build_only=args.build_only,
View
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
#
# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import os
import sys
import subprocess
import time
import yaml
import bazelci
from bazelci import DOWNSTREAM_PROJECTS
from bazelci import PLATFORMS
from bazelci import BuildkiteException
BAZEL_REPO_DIR = os.getcwd()
def bazel_culprit_finder_py_url():
"""
URL to the latest version of this script.
"""
return "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/culprit_finder.py?{}".format(int(time.time()))
def fetch_culprit_finder_py_command():
return "curl -s {0} -o culprit_finder.py".format(bazel_culprit_finder_py_url())
def get_bazel_commits_between(first_commit, second_commit):
"""
Get bazel commits between first_commit and second_commit as a list.
first_commit is not included in the list.
second_commit is included in the list.
"""
try:
output = subprocess.check_output(["git", "log", "--pretty=tformat:%H", "%s..%s"
% (first_commit, second_commit)])
return [ i for i in reversed(output.decode("utf-8").split("\n")) if i ]
except subprocess.CalledProcessError as e:
raise bazelci.BazelBuildFailedException("Failed to get bazel commits between %s..%s:\n%s"
% (first_commit, second_commit, str(e)))
def test_with_bazel_at_commit(project_name, platform_name, git_repo_location, bazel_commit):
http_config = DOWNSTREAM_PROJECTS[project_name]["http_config"]
git_repository = DOWNSTREAM_PROJECTS[project_name]["git_repository"]
return_code = bazelci.main(["runner",
"--platform=" + platform_name,
"--http_config=" + http_config,
"--git_repository=" + git_repository,
"--git_repo_location=" + git_repo_location,
"--use_bazel_at_commit=" + bazel_commit])
return return_code == 0
def start_bisecting(project_name, platform_name, commits_list):
git_repository = DOWNSTREAM_PROJECTS[project_name]["git_repository"]
git_repo_location = bazelci.clone_git_repository(git_repository, platform_name)
left = 0
right = len(commits_list)
while left < right:
mid = (left + right) // 2
mid_commit = commits_list[mid]
bazelci.print_expanded_group(":bazel: Test with Bazel built at " + mid_commit)
bazelci.eprint("Remaining suspected commits are:\n")
for i in range(left, right):
bazelci.eprint(commits_list[i] + "\n")
if test_with_bazel_at_commit(project_name, platform_name, git_repo_location, mid_commit):
bazelci.print_expanded_group(":bazel: Succeeded at " + mid_commit)
left = mid + 1
else:
bazelci.print_expanded_group(":bazel: Failed at " + mid_commit)
right = mid
bazelci.print_expanded_group(":bazel: Bisect Result")
if right == len(commits_list):
print("first bad commit not found, every commit succeeded.")
else:
first_bad_commit = commits_list[right]
print("first bad commit is " + first_bad_commit)
os.chdir(BAZEL_REPO_DIR)
bazelci.execute_command(["git", "--no-pager", "log", "-n", "1", first_bad_commit])
def print_culprit_finder_pipeline(project_name, platform_name, good_bazel_commit, bad_bazel_commit):
host_platform = PLATFORMS[platform_name].get("host-platform", platform_name)
pipeline_steps = []
command = ("%s culprit_finder.py runner --project_name=\"%s\" --platform_name=%s --good_bazel_commit=%s --bad_bazel_commit=%s"
% (bazelci.python_binary(platform_name), project_name, platform_name, good_bazel_commit, bad_bazel_commit))
pipeline_steps.append({
"label": "Bisecting for {0}".format(project_name),
"command": [
bazelci.fetch_bazelcipy_command(),
fetch_culprit_finder_py_command(),
command
],
"agents": {
"kind": "worker",
"java": PLATFORMS[platform_name]["java"],
"os": bazelci.rchop(host_platform, "_nojava", "_java8", "_java9", "_java10")
}
})
print(yaml.dump({"steps": pipeline_steps}))
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script")
subparsers = parser.add_subparsers(dest="subparsers_name")
culprit_finder = subparsers.add_parser("culprit_finder")
runner = subparsers.add_parser("runner")
runner.add_argument("--project_name", type=str)
runner.add_argument("--platform_name", type=str)
runner.add_argument("--good_bazel_commit", type=str)
runner.add_argument("--bad_bazel_commit", type=str)
args = parser.parse_args(argv)
try:
if args.subparsers_name == "culprit_finder":
try:
project_name = os.environ["PROJECT_NAME"]
platform_name = os.environ["PLATFORM_NAME"]
good_bazel_commit = os.environ["GOOD_BAZEL_COMMIT"]
bad_bazel_commit = os.environ["BAD_BAZEL_COMMIT"]
except KeyError as e:
raise BuildkiteException("Environment variable %s must be set" % str(e))
if project_name not in DOWNSTREAM_PROJECTS:
raise BuildkiteException("Project name '%s' not recognized, available projects are %s"
% (project_name, str((DOWNSTREAM_PROJECTS.keys()))))
if platform_name not in PLATFORMS:
raise BuildkiteException("Platform name '%s' not recognized, available platforms are %s"
% (platform_name, str((PLATFORMS.keys()))))
print_culprit_finder_pipeline(project_name = project_name,
platform_name = platform_name,
good_bazel_commit = good_bazel_commit,
bad_bazel_commit = bad_bazel_commit)
elif args.subparsers_name == "runner":
start_bisecting(project_name = args.project_name,
platform_name = args.platform_name,
commits_list = get_bazel_commits_between(args.good_bazel_commit, args.bad_bazel_commit))
else:
parser.print_help()
return 2
except BuildkiteException as e:
bazelci.eprint(str(e))
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

0 comments on commit 20d4560

Please sign in to comment.