Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use Docker health check to decrease wait for container #154

Merged
merged 6 commits into from
Dec 5, 2023
Merged

Use Docker health check to decrease wait for container #154

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0fed55f
Use Docker health check to decrease wait for container
philippgille Dec 3, 2023
6e27ec9
Handle error from docker inspect
philippgille Dec 3, 2023
7676d70
Return error if container isn't healthy in time
philippgille Dec 3, 2023
b99abf3
Increase wait time for containers to become healthy
philippgille Dec 3, 2023
80b65b3
Add more logs about the container status
philippgille Dec 3, 2023
dacdddf
Remove ":latest" suffix from images
philippgille Dec 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 67 additions & 29 deletions magefiles/test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,14 @@ func testImpl(impl string) (err error) {

// Implementations that require a separate service

var dockerCmd string
var dockerImage string
dockerCmd := "docker run -d --rm"
var setup func() error
// TODO: Check quoting on Windows
switch impl {
case "cockroachdb":
dockerCmd = `docker run -d --rm --name cockroachdb -p 26257:26257 --health-cmd='curl -f http://localhost:8080/health?ready=1' cockroachdb/cockroach start-single-node --insecure`
dockerImage = "cockroachdb/cockroach"
dockerCmd += ` --name cockroachdb -p 26257:26257 --health-cmd='curl -f http://localhost:8080/health?ready=1' ` + dockerImage + ` start-single-node --insecure`
setup = func() error {
var out string
out, err = script.Exec(`docker exec cockroachdb bash -c './cockroach sql --insecure --execute="create database gokv;"'`).String()
Expand All @@ -50,32 +52,44 @@ func testImpl(impl string) (err error) {
return nil
}
case "consul":
dockerCmd = `docker run -d --rm --name consul -e CONSUL_LOCAL_CONFIG='{"limits":{"http_max_conns_per_client":1000}}' -p 8500:8500 bitnami/consul`
dockerImage = "bitnami/consul"
dockerCmd += ` --name consul -e CONSUL_LOCAL_CONFIG='{"limits":{"http_max_conns_per_client":1000}}' -p 8500:8500 ` + dockerImage
case "datastore": // Google Cloud Datastore via "Cloud Datastore Emulator"
// Using the ":slim" or ":alpine" tag would require the emulator to be installed manually.
// Both ways seem to be okay for setting the project: `-e CLOUDSDK_CORE_PROJECT=gokv` and CLI parameter `--project=gokv`
// `--host-port` is required because otherwise the server only listens on localhost IN the container.
dockerCmd = `docker run -d --rm --name datastore -p 8081:8081 google/cloud-sdk gcloud beta emulators datastore start --no-store-on-disk --project=gokv --host-port=0.0.0.0:8081`
dockerImage = "google/cloud-sdk"
dockerCmd += ` --name datastore -p 8081:8081 ` + dockerImage + ` gcloud beta emulators datastore start --no-store-on-disk --project=gokv --host-port=0.0.0.0:8081`
case "dynamodb": // DynamoDB via "DynamoDB local"
dockerCmd = `docker run -d --rm --name dynamodb-local -p 8000:8000 amazon/dynamodb-local`
dockerImage = "amazon/dynamodb-local"
dockerCmd += ` --name dynamodb-local -p 8000:8000 ` + dockerImage
case "etcd":
dockerCmd = `docker run -d --rm --name etcd -p 2379:2379 --env ALLOW_NONE_AUTHENTICATION=yes --health-cmd='etcdctl endpoint health' bitnami/etcd`
dockerImage = "bitnami/etcd"
dockerCmd += ` --name etcd -p 2379:2379 --env ALLOW_NONE_AUTHENTICATION=yes --health-cmd='etcdctl endpoint health' ` + dockerImage
case "hazelcast":
dockerCmd = `docker run -d --rm --name hazelcast -p 5701:5701 --health-cmd='curl -f http://localhost:5701/hazelcast/health/node-state' hazelcast/hazelcast`
dockerImage = "hazelcast/hazelcast"
dockerCmd += ` --name hazelcast -p 5701:5701 --health-cmd='curl -f http://localhost:5701/hazelcast/health/node-state' ` + dockerImage
case "ignite":
dockerCmd = `docker run -d --rm --name ignite -p 10800:10800 --health-cmd='${IGNITE_HOME}/bin/control.sh --baseline | grep "Cluster state: active"' apacheignite/ignite`
dockerImage = "apacheignite/ignite"
dockerCmd += ` --name ignite -p 10800:10800 --health-cmd='${IGNITE_HOME}/bin/control.sh --baseline | grep "Cluster state: active"' ` + dockerImage
case "memcached":
dockerCmd = `docker run -d --rm --name memcached -p 11211:11211 memcached`
dockerImage = "memcached"
dockerCmd += ` --name memcached -p 11211:11211 ` + dockerImage
case "mongodb":
dockerCmd = `docker run -d --rm --name mongodb -p 27017:27017 --health-cmd='echo "db.runCommand({ ping: 1 }).ok" | mongosh localhost:27017/test --quiet' mongo`
dockerImage = "mongo"
dockerCmd += ` --name mongodb -p 27017:27017 --health-cmd='echo "db.runCommand({ ping: 1 }).ok" | mongosh localhost:27017/test --quiet' ` + dockerImage
case "mysql":
dockerCmd = `docker run -d --rm --name mysql -e MYSQL_ALLOW_EMPTY_PASSWORD=true -p 3306:3306 --health-cmd='mysqladmin ping -h localhost' mysql`
dockerImage = "mysql"
dockerCmd += ` --name mysql -e MYSQL_ALLOW_EMPTY_PASSWORD=true -p 3306:3306 --health-cmd='mysqladmin ping -h localhost' ` + dockerImage
case "postgresql":
dockerCmd = `docker run -d --rm --name postgres -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=gokv -p 5432:5432 --health-cmd='pg_isready -U postgres' postgres:alpine`
dockerImage = "postgres:alpine"
dockerCmd += ` --name postgres -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=gokv -p 5432:5432 --health-cmd='pg_isready -U postgres' ` + dockerImage
case "redis":
dockerCmd = `docker run -d --rm --name redis -p 6379:6379 --health-cmd='redis-cli ping' redis`
dockerImage = "redis"
dockerCmd += ` --name redis -p 6379:6379 --health-cmd='redis-cli ping' ` + dockerImage
case "s3": // Amazon S3 via Minio
dockerCmd = `docker run -d --rm --name s3 -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" -p 9000:9000 --health-cmd='mc ready local' minio/minio server /data`
dockerImage = "minio/minio"
dockerCmd += ` --name s3 -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" -p 9000:9000 --health-cmd='mc ready local' ` + dockerImage + ` server /data`
case "tablestorage": // Tablestorage via Azurite
// In the past there was this problem: https://github.com/Azure/Azurite/issues/121
// With this Docker image:
Expand All @@ -85,34 +99,40 @@ func testImpl(impl string) (err error) {
case "tablestore":
// Currently no emulator exists for Alibaba Cloud Table Store.
case "zookeeper":
dockerCmd = `docker run -d --rm --name zookeeper -p 2181:2181 -e ZOO_4LW_COMMANDS_WHITELIST=ruok --health-cmd='echo ruok | timeout 2 nc -w 2 localhost 2181 | grep imok' zookeeper`
dockerImage = "zookeeper"
dockerCmd += ` --name zookeeper -p 2181:2181 -e ZOO_4LW_COMMANDS_WHITELIST=ruok --health-cmd='echo ruok | timeout 2 nc -w 2 localhost 2181 | grep imok' ` + dockerImage
default:
return errors.New("unknown `gokv.Store` implementation")
}

// TODO: until docker images for windows appear, skip those test for windows
if dockerCmd != "" && runtime.GOOS == "windows" {
if dockerImage != "" && runtime.GOOS == "windows" {
return nil
}

// For some implementations there's no way to test with a Docker container yet.
// For them we skip the Docker stuff but still execute the tests, which can skip on connection error and we can see the skips in the test results.
if dockerCmd != "" {
// Start Docker container
if dockerImage != "" {
// Pull Docker image
fmt.Printf("Pulling Docker image %s...", dockerImage)
var out string
out, err = script.Exec(dockerCmd).String()
out, err = script.Exec("docker pull " + dockerImage).String()
if err != nil {
// Depending on the error, printing the output could be interesting, as it could be more info than what's in err.
fmt.Println(out)
return err
}
// In the success case, if the image was pulled as part of `docker run`, the output is not only the container ID, but also the pull progress.
outLines := strings.Split(out, "\n")
containerID := outLines[len(outLines)-1]
// Docker output could end with a newline, in which case we use the previous line
if containerID == "" {
containerID = outLines[len(outLines)-2]
// Start Docker container
fmt.Println("Starting Docker container...")
fmt.Println(dockerCmd)
out, err = script.Exec(dockerCmd).String()
if err != nil {
fmt.Println(out)
return err
}
// Thanks to separate pull and run, the output of the run is only the container ID
// instead of including pull progress.
containerID := strings.ReplaceAll(out, "\n", "")
defer func() {
out, err2 := script.Exec("docker stop " + containerID).String()
if err2 != nil {
Expand All @@ -127,10 +147,28 @@ func testImpl(impl string) (err error) {
}()

// Wait for container to be started
// TODO: Use a proper health/startup check for this, as many services are ready much quicker than 10s.
time.Sleep(10 * time.Second)
out, _ = script.Exec("docker inspect --format='{{.State.Health.Status}}' " + containerID).String()
fmt.Println(out)
if strings.Contains(dockerCmd, "--health-cmd") {
for i := 0; i < 60; i++ {
out, err = script.Exec("docker inspect --format='{{.State.Health.Status}}' " + containerID).String()
if err != nil {
fmt.Println(out)
return err
}
out = strings.ReplaceAll(out, "\n", "")
if out == "healthy" {
break
}
fmt.Printf("Waiting for container to be healthy... (%d/60)\n", i+1)
time.Sleep(time.Second)
}
// Return an error if the container isn't healthy yet
if out != "healthy" {
return errors.New("container didn't become healthy in time")
}
} else {
fmt.Println("Waiting 10 seconds for container due to missing health check...")
time.Sleep(10 * time.Second)
}

if setup != nil {
err = setup()
Expand Down
Loading