Skip to content

Commit

Permalink
DockerCompose: wait logic inspect yaml
Browse files Browse the repository at this point in the history
  • Loading branch information
@rukai
rukai committed Nov 17, 2022
1 parent 65066c2 commit 6891895
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 69 deletions.
22 changes: 21 additions & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions test-helpers/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@ anyhow = "1.0.42"
nix = "0.25.0"
rcgen = "0.10.0"
docker-api = "0.12.0"
serde_yaml = "0.9.0"
203 changes: 135 additions & 68 deletions test-helpers/src/docker_compose.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
use anyhow::{anyhow, Result};
use docker_api::{Container, Docker};
use serde_yaml::Value;
use std::collections::HashMap;
use std::fmt::Write;
use std::io::ErrorKind;
use std::process::Command;
use std::time::{self, Duration};
Expand Down Expand Up @@ -45,17 +48,16 @@ impl DockerCompose {
/// Creates a new DockerCompose object by submitting a file to the underlying docker-compose
/// system. Executes `docker-compose -f [file_path] up -d`
///
/// Will spin until it detects all the containers have started up.
/// This logic is implemented internally per docker-compose file.
/// If a docker-compose file is used that hasnt had this logic implemented for it yet
/// Will spin until it detects all the containers have started up by inspecting the logs for a magic string.
/// This logic is implemented internally per docker image name.
/// If a service uses an image that hasnt had this logic implemented for it yet
/// a panic will occur instructing the developer to implement this logic.
///
/// # Arguments
/// * `file_path` - The path to the docker-compose yaml file.
///
/// # Panics
/// * Will panic if docker-compose is not installed
///
pub fn new(file_path: &str) -> Self {
if let Err(ErrorKind::NotFound) = Command::new("docker-compose")
.output()
Expand Down Expand Up @@ -131,65 +133,92 @@ impl DockerCompose {
}

fn wait_for_containers_to_startup(&self) {
match self.file_path.as_ref() {
"tests/transforms/docker-compose-moto.yaml" => {
self.wait_for_log("Press CTRL+C to quit", 1)
}
"example-configs/redis-passthrough/docker-compose.yaml"
| "example-configs/redis-tls/docker-compose.yaml" => {
self.wait_for_log("Ready to accept connections", 1)
}
"example-configs/redis-multi/docker-compose.yaml" => {
self.wait_for_log("Ready to accept connections", 3)
}
"tests/test-configs/redis-cluster-ports-rewrite/docker-compose.yaml"
| "tests/test-configs/redis-cluster-auth/docker-compose.yaml"
| "example-configs/redis-cluster-handling/docker-compose.yaml"
| "example-configs/redis-cluster-hiding/docker-compose.yaml"
| "example-configs/redis-cluster-tls/docker-compose.yaml"
| "example-configs/redis-cluster-tls/docker-compose-with-key.yaml" => {
self.wait_for_log("Cluster state changed", 6)
}
"example-configs/redis-cluster-dr/docker-compose.yaml" => {
self.wait_for_log("Cluster state changed", 12)
}
"example-configs/cassandra-passthrough/docker-compose.yaml"
| "example-configs/cassandra-tls/docker-compose.yaml"
| "example-configs/cassandra-redis-cache/docker-compose.yaml"
| "example-configs/cassandra-protect-local/docker-compose.yaml"
| "example-configs/cassandra-protect-aws/docker-compose.yaml"
| "example-configs/cassandra-request-throttling/docker-compose.yaml"
| "tests/test-configs/cassandra-passthrough-parse-request/docker-compose.yaml"
| "tests/test-configs/cassandra-passthrough-parse-response/docker-compose.yaml" => {
self.wait_for_log("Startup complete", 1)
}
"tests/test-configs/cassandra-peers-rewrite/docker-compose-4.0-cassandra.yaml"
| "tests/test-configs/cassandra-peers-rewrite/docker-compose-3.11-cassandra.yaml" => {
self.wait_for_log("Startup complete", 2)
}
"example-configs-docker/cassandra-peers-rewrite/docker-compose.yaml"
| "example-configs/cassandra-cluster/docker-compose-cassandra-v4.yaml"
| "example-configs/cassandra-cluster/docker-compose-cassandra-v3.yaml"
| "example-configs/cassandra-cluster-multi-rack/docker-compose.yaml"
| "example-configs/cassandra-cluster-tls/docker-compose.yaml" => {
self.wait_for_log("Startup complete", 3)
}
path => unimplemented!(
"Unknown compose file `{path}` Please implement waiting logic for it here.",
),
let images = [
Image {
name: "motoserver/moto".to_owned(),
log_to_wait_for: "Press CTRL+C to quit".to_owned(),
},
Image {
name: "library/redis:5.0.9".to_owned(),
log_to_wait_for: "Ready to accept connections".to_owned(),
},
Image {
name: "docker.io/bitnami/redis-cluster:6.0-debian-10".to_owned(),
log_to_wait_for: "Cluster state changed".to_owned(),
},
Image {
name: "bitnami/redis-cluster:6.0-debian-10".to_owned(),
log_to_wait_for: "Cluster state changed".to_owned(),
},
Image {
name: "shotover-int-tests/cassandra:4.0.6".to_owned(),
log_to_wait_for: "Startup complete".to_owned(),
},
Image {
name: "shotover-int-tests/cassandra-tls:4.0.6".to_owned(),
log_to_wait_for: "Startup complete".to_owned(),
},
Image {
name: "shotover-int-tests/cassandra:3.11.13".to_owned(),
log_to_wait_for: "Startup complete".to_owned(),
},
];

let services: Vec<Service> = self
.get_service_to_image()
.into_iter()
.map(
|(service_name, image_name)| match images.iter().find(|image| image.name == image_name) {
Some(image) => Service {
name: service_name,
log_to_wait_for: image.log_to_wait_for.clone(),
},
None => panic!("DockerCompose does not yet know about the image {image_name}, please add it to the list above."),
},
)
// TODO: ugh..
.filter(|x| !x.name.contains("init"))
.collect();

self.wait_for_logs(&services);
}

fn get_service_to_image(&self) -> HashMap<String, String> {
// If we got this far then docker-compose already succesfully parsed the docker-compose.yaml,
// so our error reporting only has to be good enough to debug issues in our implementation.
let compose_yaml: Value =
serde_yaml::from_str(&std::fs::read_to_string(&self.file_path).unwrap()).unwrap();
let mut result = HashMap::new();
match compose_yaml {
Value::Mapping(root) => match root.get("services").unwrap() {
Value::Mapping(services) => {
for (service_name, service) in services {
let service_name = match service_name {
Value::String(service_name) => service_name,
service_name => panic!("Unexpected service_name {service_name:?}"),
};
match service {
Value::Mapping(service) => {
let image = match service.get("image").unwrap() {
Value::String(image) => image,
image => panic!("Unexpected image {image:?}"),
};
result.insert(service_name.clone(), image.clone());
}
service => panic!("Unexpected service {service:?}"),
}
}
}
services => panic!("Unexpected services {services:?}"),
},
root => panic!("Unexpected root {root:?}"),
}
result
}

/// Waits for a string to appear in the docker-compose log output `count` times within `time` seconds.
///
/// # Arguments
/// * `log_to_wait_for` - A regular expression defining the text to find in the docker-container log
/// output.
/// * `count` - The number of times the regular expression should be found.
///
/// # Panics
/// * If `count` occurrences of `log_text` is not found in the log within 120 seconds.
fn wait_for_log(&self, log_to_wait_for: &str, count: usize) {
/// Wait until the requirements in every Service is met.
/// Will panic if a timeout occurs.
fn wait_for_logs(&self, services: &[Service]) {
let timeout = Duration::from_secs(120);

// TODO: remove this check once CI docker-compose is updated (probably ubuntu 22.04)
Expand All @@ -200,24 +229,52 @@ impl DockerCompose {

let instant = time::Instant::now();
loop {
// check if the service is completely ready
let full_log = run_command("docker-compose", &["-f", &self.file_path, "logs"]).unwrap();
let current_count = full_log.matches(log_to_wait_for).count();
if current_count >= count {
// check if every service is completely ready
if services.iter().all(|service| {
let log = run_command(
"docker-compose",
&["-f", &self.file_path, "logs", &service.name],
)
.unwrap();
log.contains(&service.log_to_wait_for)
}) {
return;
}

let all_logs = run_command("docker-compose", &["-f", &self.file_path, "logs"]).unwrap();

// check if the service has failed in some way
// this allows us to report the failure to the developer a lot sooner than just relying on the timeout
if can_use_status_flag {
self.assert_no_containers_in_service_with_status("exited", &full_log);
self.assert_no_containers_in_service_with_status("dead", &full_log);
self.assert_no_containers_in_service_with_status("removing", &full_log);
self.assert_no_containers_in_service_with_status("exited", &all_logs);
self.assert_no_containers_in_service_with_status("dead", &all_logs);
self.assert_no_containers_in_service_with_status("removing", &all_logs);
}

// if all else fails timeout the wait
if instant.elapsed() > timeout {
panic!("wait_for_log {timeout:?} timer expired. Found {current_count}/{count} instances of '{log_to_wait_for}' in the log\n{full_log}");
let mut results = "".to_owned();
for service in services {
let log = run_command(
"docker-compose",
&["-f", &self.file_path, "logs", &service.name],
)
.unwrap();
let found = if log.contains(&service.log_to_wait_for) {
"Found"
} else {
"Missing"
};

writeln!(
results,
"* Service {}, searched for '{}', was {}",
service.name, service.log_to_wait_for, found
)
.unwrap();
}

panic!("wait_for_log {timeout:?} timer expired. Results:\n{results}\nLogs:\n{all_logs}");
}
}
}
Expand Down Expand Up @@ -289,6 +346,16 @@ impl DockerCompose {
}
}

struct Image {
name: String,
log_to_wait_for: String,
}

struct Service {
name: String,
log_to_wait_for: String,
}

impl Drop for DockerCompose {
fn drop(&mut self) {
if std::thread::panicking() {
Expand Down

0 comments on commit 6891895

Please sign in to comment.