Skip to content

Commit

Permalink
feat(bolt): run tests in containers (#947)
Browse files Browse the repository at this point in the history 
<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->
  • Loading branch information
@MasterPtato
MasterPtato committed Jun 26, 2024
1 parent 7ebe1f1 commit 08a53e3
Show file tree
Hide file tree
Showing 6 changed files with 341 additions and 117 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ Bolt.local.toml

# Generated code
gen/build_script.sh
gen/test_build_script.sh
gen/svc/
gen/tf/
gen/docker/
gen/tests/
gen/k8s/

# Rust
Expand Down
1 change: 0 additions & 1 deletion infra/misc/svc_scripts/install_ca.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,3 @@ exec >> "/var/log/install-ca.txt" 2>&1
#
# Overriding LD_LIBRARY_PATH prevents apt from using the OpenSSL installation from /nix/store (if mounted).
LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib update-ca-certificates

309 changes: 263 additions & 46 deletions lib/bolt/core/src/dep/cargo/cli.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,21 @@ use anyhow::{ensure, Context, Result};
use indoc::{formatdoc, indoc};
use regex::Regex;
use serde_json::json;
use tokio::{fs, io::AsyncReadExt, process::Command, task::block_in_place};
use tokio::{fs, process::Command, task::block_in_place};
use uuid::Uuid;

use crate::{config, context::ProjectContext};
use crate::context::ProjectContext;

const DOCKERIGNORE: &str = indoc!(
r#"
*
# Rivet
!Bolt.toml
!infra/misc/svc_scripts
!oss/infra/misc/svc_scripts
!gen/docker
!gen/build_script.sh
!gen/test_build_script.sh
sdks/runtime
oss/sdks/runtime
!lib
Expand Down Expand Up @@ -184,18 +188,18 @@ pub async fn build<'a, T: AsRef<str>>(ctx: &ProjectContext, opts: BuildOpts<'a,
COPY . .
COPY {build_script_path} build_script.sh
# Build and copy all binaries from target directory into an empty image (it is not
# included in the output because of cache mount)
RUN chmod +x ./build_script.sh
RUN \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/rivet/target \
--mount=type=cache,target=/usr/rivet/oss/target \
sh -c ./build_script.sh && mkdir /usr/bin/rivet && find target/{optimization} -maxdepth 1 -type f ! -name "*.*" -exec mv {{}} /usr/bin/rivet/ \;
# Copy all binaries from target directory into an empty image (it is not included in
# the output because of cache mount)
# Create an empty image and copy binaries to it (this is to minimize the size of the image)
# Create an empty image and copy binaries + test outputs to it (this is to minimize the
# size of the image)
FROM scratch
COPY --from=rust /usr/bin/rivet/ /
"#,
Expand Down Expand Up @@ -300,8 +304,12 @@ pub async fn build<'a, T: AsRef<str>>(ctx: &ProjectContext, opts: BuildOpts<'a,
Ok(())
}

pub const TEST_IMAGE_NAME: &str = "test";

pub struct BuildTestOpts<'a, T: AsRef<str>> {
pub build_calls: Vec<BuildTestCall<'a, T>>,
/// Builds for release mode.
pub release: bool,
/// How many threads to run in parallel when building.
pub jobs: Option<usize>,
pub test_filters: &'a [String],
Expand All @@ -324,44 +332,220 @@ pub async fn build_tests<'a, T: AsRef<str>>(
ctx: &ProjectContext,
opts: BuildTestOpts<'a, T>,
) -> Result<Vec<TestBinary>> {
let jobs_flag = if let Some(jobs) = opts.jobs {
format!("--jobs {jobs}")
} else {
String::new()
};

let build_calls =
opts.build_calls
.iter()
.map(|build_call| {
let path = build_call.path.display();
let package_flags = build_call
.packages
.iter()
.map(|x| format!("--package {}", x.as_ref()))
.collect::<Vec<_>>()
.join(" ");
// Generate a name from the build call path
let name = build_call.path.iter()
.map(|s| s.to_string_lossy())
.collect::<Vec<_>>()
.join("-");

format!("(cd {path} && cargo test --no-run --message-format=json-render-diagnostics {jobs_flag} {package_flags}) > gen/tests/{name}.out")
})
.collect::<Vec<_>>()
.join("\n");

let build_script_path = ctx.gen_path().join("test_build_script.sh");
let build_script_path_relative = build_script_path
.strip_prefix(ctx.path())
.context("failed to strip prefix")?;

// TODO: Not sure why the .cargo/config.toml isn't working with nested projects, have to hardcode
// the target dir
// Generate build script
let build_script = formatdoc!(
r#"
#!/bin/bash
set -euf
[ -z "${{CARGO_TARGET_DIR+x}}" ] && export CARGO_TARGET_DIR=$(readlink -f ./target)
# Used for Tokio Console. See https://github.com/tokio-rs/console#using-it
export RUSTFLAGS="--cfg tokio_unstable"
{build_calls}
"#,
);

// Write build script to file
fs::write(&build_script_path, build_script).await?;

let mut test_binaries = vec![];
for build_call in opts.build_calls {
let abs_path = ctx.path().join(build_call.path);

// Build command
let mut cmd = Command::new("cargo");
cmd.args(&[
"test",
"--no-run",
"--message-format=json-render-diagnostics",
])
.stdout(std::process::Stdio::piped())
.current_dir(abs_path)
// Used for Tokio Console. See https://github.com/tokio-rs/console#using-it
.env("RUSTFLAGS", "--cfg tokio_unstable");
if let Some(jobs) = opts.jobs {
cmd.args(&["--jobs", &jobs.to_string()]);
}
for test in build_call.packages {
cmd.args(&["--package", test.as_ref()]);
}
if std::env::var("CARGO_TARGET_DIR").is_err() {
cmd.env("CARGO_TARGET_DIR", ctx.cargo_target_dir());
}
let mut child = cmd.spawn()?;

// Capture stdout
let mut stdout = child.stdout.take().context("missing stdout")?;
let mut stdout_str = String::new();
stdout.read_to_string(&mut stdout_str).await?;
// Execute build command
let temp_container_name = if ctx.build_svcs_locally() {
// Create directory for test outputs
fs::create_dir_all(ctx.gen_path().join("tests")).await?;

// Make build script executable
let mut cmd = Command::new("chmod");
cmd.current_dir(ctx.path());
cmd.arg("+x");
cmd.arg(build_script_path.display().to_string());
let status = cmd.status().await?;
ensure!(status.success());

// Execute
let mut cmd = Command::new(build_script_path.display().to_string());
cmd.current_dir(ctx.path());
let status = cmd.status().await?;
ensure!(status.success());

None
} else {
let optimization = if opts.release { "release" } else { "debug" };
// Get repo to push to
let (push_repo, _) = ctx.docker_repos().await;
let source_hash = ctx.source_hash();

// Create directory for docker files
let gen_path = ctx.gen_path().join("docker");
fs::create_dir_all(&gen_path).await?;

let image_tag = format!("{push_repo}{TEST_IMAGE_NAME}:{source_hash}");
let dockerfile_path = gen_path.join(format!("Dockerfile.test_build"));

// Resolve the symlink for the svc_scripts dir since Docker does not resolve
// symlinks in COPY commands
let infra_path = ctx.path().join("infra");
let infra_path_resolved = tokio::fs::read_link(&infra_path)
.await
.map_or_else(|_| infra_path.clone(), |path| ctx.path().join(path));
let svc_scripts_path = infra_path_resolved.join("misc").join("svc_scripts");
let svc_scripts_path_relative = svc_scripts_path
.strip_prefix(ctx.path())
.context("failed to strip prefix")?;

// See above `build` fn for more info
fs::write(
&dockerfile_path,
formatdoc!(
r#"
# syntax=docker/dockerfile:1.2
FROM rust:1.77.2-slim AS build
RUN apt-get update && apt-get install -y protobuf-compiler pkg-config libssl-dev g++ git
RUN apt-get install --yes libpq-dev wget
RUN wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz \
&& tar xzf sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz \
&& mv sccache-v0.2.15-x86_64-unknown-linux-musl/sccache /usr/local/bin/sccache \
&& chmod +x /usr/local/bin/sccache
WORKDIR /usr/rivet
COPY . .
COPY {build_script_path} build_script.sh
# TODO: Only copy test binaries (currently copies all binaries in target/opt/deps)
# Build and copy all binaries from target directory into an empty image (it is not
# included in the output because of cache mount)
RUN chmod +x ./build_script.sh
RUN mkdir -p gen/tests
RUN \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/rivet/target \
--mount=type=cache,target=/usr/rivet/oss/target \
sh -c ./build_script.sh && mkdir /usr/bin/rivet && find target/{optimization}/deps -maxdepth 1 -type f ! -name "*.*" -exec mv {{}} /usr/bin/rivet/ \;
FROM debian:12.1-slim AS run
# Update ca-certificates. Install curl for health checks.
RUN DEBIAN_FRONTEND=noninteractive apt-get update -y && apt-get install -y --no-install-recommends ca-certificates openssl curl
# Copy supporting scripts
COPY {svc_scripts_path}/health_check.sh {svc_scripts_path}/install_ca.sh /usr/bin/
RUN chmod +x /usr/bin/health_check.sh /usr/bin/install_ca.sh
# Copy generated test outputs
COPY --from=build /usr/rivet/gen/tests/ /usr/rivet/gen/tests/
# Copy final binaries
COPY --from=build /usr/bin/rivet/ /usr/bin/rivet/
"#,
build_script_path = build_script_path_relative.display(),
svc_scripts_path = svc_scripts_path_relative.display(),
),
)
.await?;

let dockerignore_path = gen_path.join("Dockerfile.test_build.dockerignore");
fs::write(&dockerignore_path, DOCKERIGNORE).await?;

// Build image
let mut cmd = Command::new("docker");
cmd.env("DOCKER_BUILDKIT", "1");
cmd.current_dir(ctx.path());
cmd.arg("build");
cmd.arg("-f").arg(dockerfile_path);
// Prints plain console output for debugging
// cmd.arg("--progress=plain");
cmd.arg("-t").arg(&image_tag);
cmd.arg(".");

let status = cmd.status().await?;
ensure!(status.success(), "failed to run build command");

// TODO: Find better way to copy files from image?
let temp_container_name = Uuid::new_v4().to_string();
let output_path = ctx.gen_path().display().to_string();

// Create the temporary container
let mut cmd = Command::new("docker");
cmd.current_dir(ctx.path());
cmd.arg("run");
cmd.arg("--rm");
cmd.arg("-d");
cmd.arg("--name").arg(&temp_container_name);
cmd.arg(&image_tag);
cmd.arg("sleep").arg("120");

let output = cmd.output().await?;
ensure!(output.status.success(), "failed to run temp container");

// Copy the files from the temporary container
let mut cmd = Command::new("docker");
cmd.current_dir(ctx.path());
cmd.arg("cp")
.arg(format!("{}:/usr/rivet/gen/tests", temp_container_name))
.arg(output_path);

let status = cmd.status().await?;
ensure!(status.success(), "failed to copy files from container");

Some(temp_container_name)
};

for build_call in opts.build_calls {
let name = build_call
.path
.iter()
.map(|s| s.to_string_lossy())
.collect::<Vec<_>>()
.join("-");
let output_path = ctx.gen_path().join("tests").join(format!("{name}.out"));

// Wait for finish
let status = child.wait().await?;
ensure!(status.success(), "build test failed");
let stdout = fs::read_to_string(output_path).await?;

// Parse artifacts
let test_count_re = Regex::new(r"(?m)^(.*): test$").unwrap();
for line in stdout_str.lines() {
for line in stdout.lines() {
let v = serde_json::from_str::<serde_json::Value>(line).context("invalid json")?;
if v["reason"] == "compiler-artifact" && v["target"]["kind"] == json!(["test"]) {
if let Some(executable) = v["filenames"][0].as_str() {
Expand All @@ -382,21 +566,54 @@ pub async fn build_tests<'a, T: AsRef<str>>(
.context("missing target name")?;

// Parse the test count from the binary
let test_list_args = [
&["--list".to_string(), "--format".into(), "terse".into()],
opts.test_filters,
]
.concat();
let test_list_stdout =
block_in_place(|| duct::cmd(executable, &test_list_args).read())?;
let (exec_path, test_list_stdout) =
if let Some(temp_container_name) = &temp_container_name {
let exec_name = Path::new(executable);
let exec_path = Path::new("/usr/bin/rivet")
.join(exec_name.file_name().expect("no file name"));

let test_list_args = [
&[
"exec".to_string(),
temp_container_name.clone(),
exec_path.display().to_string(),
"--list".into(),
"--format".into(),
"terse".into(),
],
opts.test_filters,
]
.concat();

(
exec_path,
block_in_place(|| duct::cmd("docker", &test_list_args).read())?,
)
} else {
// Make path relative to project
let relative_path = Path::new(executable)
.strip_prefix(ctx.cargo_target_dir())
.context(format!("path not in project: {executable}"))?;

let test_list_args = [
&["--list".to_string(), "--format".into(), "terse".into()],
opts.test_filters,
]
.concat();

(
relative_path.to_path_buf(),
block_in_place(|| duct::cmd(executable, &test_list_args).read())?,
)
};

// Run a test container for every test in the binary
for cap in test_count_re.captures_iter(&test_list_stdout) {
let test_name = &cap[1];
test_binaries.push(TestBinary {
package: package.to_string(),
target: target.to_string(),
path: PathBuf::from(executable),
path: exec_path.clone(),
test_name: test_name.to_string(),
})
}
Expand Down
Loading

0 comments on commit 08a53e3

Please sign in to comment.