Skip to content

Commit

Permalink
Ensure Docker container logs are captured on error
Browse files Browse the repository at this point in the history
If a Docker containers fails to start, the logs were not getting picked up correctly. With this fix the logs are grabbed _before_ the container is stopped, which loses the logs.

Tests added to ensure this works as expected.
  • Loading branch information
big-andy-coates committed Mar 7, 2023
1 parent ba8a508 commit e58ee2c
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 21 deletions.
17 changes: 1 addition & 16 deletions buildSrc/src/main/kotlin/creek-common-convention.gradle.kts
Expand Up @@ -20,6 +20,7 @@
* <p>Apply to all java modules, usually excluding the root project in multi-module sets.
*
* <p>Version: 1.5
* - 1.6: Remove GitHub packages for snapshots
* - 1.5: Add filters to exclude generated sources
* - 1.4: Add findsecbugs-plugin
* - 1.3: Fail on warnings for test code too.
Expand All @@ -42,29 +43,13 @@ java {
repositories {
mavenCentral()

// Primary snapshot repo:
maven {
url = uri("https://s01.oss.sonatype.org/content/repositories/snapshots/")
mavenContent {
includeGroup("org.creekservice")
snapshotsOnly()
}
}

// Backup snapshot repo:
maven {
url = uri("https://maven.pkg.github.com/creek-service/*")
credentials {
username = System.getenv("GITHUB_ACTOR")
password = System.getenv("GITHUB_TOKEN")
}
mavenContent {
includeGroup("org.creekservice")
snapshotsOnly()
}
}

mavenCentral()
}

dependencies {
Expand Down
Expand Up @@ -123,8 +123,9 @@ public void start() {
imageName,
container.getContainerId());
} catch (final Exception e) {
final String logs = container.getLogs();
stop();
throw new FailedToStartServiceException(name, imageName, container, e);
throw new FailedToStartServiceException(name, imageName, logs, e);
}
}

Expand Down Expand Up @@ -343,7 +344,7 @@ private static final class FailedToStartServiceException extends RuntimeExceptio
FailedToStartServiceException(
final String name,
final DockerImageName imageName,
final GenericContainer<?> container,
final String logs,
final Throwable cause) {
super(
"Failed to start service: "
Expand All @@ -355,7 +356,7 @@ private static final class FailedToStartServiceException extends RuntimeExceptio
+ cause.getMessage()
+ lineSeparator()
+ "Logs: "
+ container.getLogs(),
+ logs,
cause);
}
}
Expand Down
Expand Up @@ -18,6 +18,7 @@

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNullElse;
import static org.creekservice.api.observability.lifecycle.LoggableLifecycle.SERVICE_TYPE;
import static org.creekservice.api.test.hamcrest.AssertEventually.assertThatEventually;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
Expand All @@ -30,6 +31,7 @@
import static org.hamcrest.Matchers.startsWith;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
Expand All @@ -46,10 +48,13 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
import org.creekservice.api.observability.lifecycle.BasicLifecycle;
import org.creekservice.api.platform.metadata.ServiceDescriptor;
import org.creekservice.api.system.test.executor.ExecutorOptions.MountInfo;
import org.creekservice.api.system.test.extension.component.definition.ServiceDefinition;
Expand All @@ -68,6 +73,7 @@
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.quality.Strictness;
import org.mockito.stubbing.Answer;
import org.testcontainers.DockerClientFactory;

@SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
Expand Down Expand Up @@ -226,7 +232,7 @@ void shouldRestartService() {
}

@Test
void shouldThrowOnServiceStartFailure() {
void shouldThrowOnUnknownDockerImage() {
// Given:
when(serviceDef.dockerImage()).thenReturn("i-do-not-exist");
final ServiceInstance instance = instances.add(serviceDef);
Expand All @@ -239,7 +245,32 @@ void shouldThrowOnServiceStartFailure() {
e.getMessage(),
startsWith(
"Failed to start service: test-service-0, image: i-do-not-exist:latest"));
assertThat(e.getMessage(), containsString("Cause: Container startup failed"));
assertThat(e.getCause().getMessage(), containsString("Container startup failed"));

assertThat(e.getCause().getCause().getMessage(), containsString("Can't get Docker image"));
}

@Test
void shouldThrowOnServiceStartFailure() {
// Given:
doAnswer(tellServiceToFail()).when(serviceDef).configureInstance(any());
final ServiceInstance instance = instances.add(serviceDef);

// When:
final Exception e = assertThrows(RuntimeException.class, instance::start);

// Then:
assertThat(
e.getMessage(),
startsWith(
"Failed to start service: test-service-0, image: "
+ SERVICE_IMAGE
+ ":latest"));
assertThat(
e.getMessage(),
containsString(
"CREEK_SERVICE_SHOULD_FAIL is set, so this service is going bye-byes!"));
assertThat(e.getMessage(), containsString("Service going BOOM!"));
}

@Test
Expand Down Expand Up @@ -571,6 +602,21 @@ private ContainerConfig instanceConfig(final ServiceInstance instance) {
.getConfig();
}

private Answer<Void> tellServiceToFail() {
return inv -> {
final ConfigurableServiceInstance instance = inv.getArgument(0);
instance.addEnv("CREEK_SERVICE_SHOULD_FAIL", "true")
.setStartupLogMessage(
".*"
+ Pattern.quote(BasicLifecycle.started.logMessage(SERVICE_TYPE))
+ ".*",
1)
.setStartupAttempts(1)
.setStartupTimeout(Duration.ofSeconds(3));
return null;
};
}

private static MountInfo mount(
final Path hostPath, final Path containerPath, final boolean readOnly) {
final MountInfo mount =
Expand Down
Expand Up @@ -32,6 +32,7 @@ private ServiceMain() {}

public static void main(final String... args) {
dumpEnv();
maybeFail();
logMount();
doLogging();
awaitShutdown();
Expand All @@ -43,6 +44,13 @@ private static void dumpEnv() {
.forEach(e -> LOGGER.info("Env: " + e.getKey() + "=" + e.getValue()));
}

private static void maybeFail() {
if (System.getenv("CREEK_SERVICE_SHOULD_FAIL") != null) {
LOGGER.error("CREEK_SERVICE_SHOULD_FAIL is set, so this service is going bye-byes!");
throw new RuntimeException("Service going BOOM!");
}
}

@SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
private static void logMount() {
final Path mount = Paths.get("/opt/creek/test_mount");
Expand Down

0 comments on commit e58ee2c

Please sign in to comment.