Skip to content

HIVE-27295: Improve docker logging in AbstractExternalDB #4268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 27, 2023
Merged

HIVE-27295: Improve docker logging in AbstractExternalDB #4268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 25 additions & 14 deletions itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,12 @@ private String[] buildLogCmd() {
return new String[] { "docker", "logs", getDockerContainerName() };
}


private ProcessResults runCmd(String[] cmd, long secondsToWait)
throws IOException, InterruptedException {
LOG.info("Going to run: " + String.join(" ", cmd));
Process proc = Runtime.getRuntime().exec(cmd);
if (!proc.waitFor(secondsToWait, TimeUnit.SECONDS)) {
throw new RuntimeException(
"Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds");
if (!proc.waitFor(Math.abs(secondsToWait), TimeUnit.SECONDS)) {
throw new RuntimeException("Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds");
}
BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
final StringBuilder lines = new StringBuilder();
Expand All @@ -99,41 +97,54 @@ private ProcessResults runCmd(String[] cmd, long secondsToWait)
reader = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
final StringBuilder errLines = new StringBuilder();
reader.lines().forEach(s -> errLines.append(s).append('\n'));
LOG.info("Result size: " + lines.length() + ";" + errLines.length());
LOG.info("Result lines#: {}(stdout);{}(stderr)",lines.length(), errLines.length());
return new ProcessResults(lines.toString(), errLines.toString(), proc.exitValue());
}

private int runCmdAndPrintStreams(String[] cmd, long secondsToWait)
private ProcessResults runCmdAndPrintStreams(String[] cmd, long secondsToWait)
throws InterruptedException, IOException {
ProcessResults results = runCmd(cmd, secondsToWait);
LOG.info("Stdout from proc: " + results.stdout);
LOG.info("Stderr from proc: " + results.stderr);
return results.rc;
return results;
}


public void launchDockerContainer() throws Exception {
runCmdAndPrintStreams(buildRmCmd(), 600);
if (runCmdAndPrintStreams(buildRunCmd(), 600) != 0) {
throw new RuntimeException("Unable to start docker container");
if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) {
printDockerEvents();
throw new RuntimeException("Unable to start docker container");
}
long startTime = System.currentTimeMillis();
ProcessResults pr;
do {
Thread.sleep(1000);
pr = runCmd(buildLogCmd(), 30);
pr = runCmdAndPrintStreams(buildLogCmd(), 30);
if (pr.rc != 0) {
throw new RuntimeException("Failed to get docker logs");
printDockerEvents();
throw new RuntimeException("Failed to get docker logs");
}
} while (startTime + MAX_STARTUP_WAIT >= System.currentTimeMillis() && !isContainerReady(pr));
if (startTime + MAX_STARTUP_WAIT < System.currentTimeMillis()) {
throw new RuntimeException("Container failed to be ready in " + MAX_STARTUP_WAIT/1000 +
" seconds");
printDockerEvents();
throw new RuntimeException(
String.format("Container initialization failed within %d seconds. Please check the hive logs.",
MAX_STARTUP_WAIT / 1000));
}
}

protected void printDockerEvents() {
try {
runCmdAndPrintStreams(new String[] { "docker", "events", "--since", "24h", "--until", "0s" }, 3);
} catch (Exception e) {
LOG.warn("A problem was encountered while attempting to retrieve Docker events (the system made an analytical"
+ " best effort to list the events to reveal the root cause). No further actions are necessary.", e);
}
}

public void cleanupDockerContainer() throws IOException, InterruptedException {
if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) {
if (runCmdAndPrintStreams(buildRmCmd(), 600).rc != 0) {
throw new RuntimeException("Unable to remove docker container");
}
}
Expand Down
34 changes: 23 additions & 11 deletions ...e-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,21 +125,25 @@ public ProcessResults(String stdout, String stderr, int rc) {
@Override
public void before() throws Exception { //runDockerContainer
runCmdAndPrintStreams(buildRmCmd(), 600);
if (runCmdAndPrintStreams(buildRunCmd(), 600) != 0) {
if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) {
printDockerEvents();
throw new RuntimeException("Unable to start docker container");
}
long startTime = System.currentTimeMillis();
ProcessResults pr;
do {
Thread.sleep(1000);
pr = runCmd(buildLogCmd(), 30);
pr = runCmdAndPrintStreams(buildLogCmd(), 30);
if (pr.rc != 0) {
printDockerEvents();
throw new RuntimeException("Failed to get docker logs");
}
} while (startTime + MAX_STARTUP_WAIT >= System.currentTimeMillis() && !isContainerReady(pr));
if (startTime + MAX_STARTUP_WAIT < System.currentTimeMillis()) {
throw new RuntimeException("Container failed to be ready in " + MAX_STARTUP_WAIT/1000 +
" seconds");
printDockerEvents();
throw new RuntimeException(
String.format("Container initialization failed within %d seconds. Please check the hive logs.",
MAX_STARTUP_WAIT / 1000));
}
MetastoreSchemaTool.setHomeDirForTesting();
}
Expand All @@ -152,7 +156,7 @@ public void after() { // stopAndRmDockerContainer
return;
}
try {
if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) {
if (runCmdAndPrintStreams(buildRmCmd(), 600).rc != 0) {
throw new RuntimeException("Unable to remove docker container");
}
} catch (InterruptedException | IOException e) {
Expand All @@ -174,9 +178,8 @@ private ProcessResults runCmd(String[] cmd, long secondsToWait)
throws IOException, InterruptedException {
LOG.info("Going to run: " + StringUtils.join(cmd, " "));
Process proc = Runtime.getRuntime().exec(cmd);
if (!proc.waitFor(secondsToWait, TimeUnit.SECONDS)) {
throw new RuntimeException(
"Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds");
if (!proc.waitFor(Math.abs(secondsToWait), TimeUnit.SECONDS)) {
throw new RuntimeException("Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds");
}
BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
final StringBuilder lines = new StringBuilder();
Expand All @@ -185,16 +188,25 @@ private ProcessResults runCmd(String[] cmd, long secondsToWait)
reader = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
final StringBuilder errLines = new StringBuilder();
reader.lines().forEach(s -> errLines.append(s).append('\n'));
LOG.info("Result size: " + lines.length() + ";" + errLines.length());
LOG.info("Result lines#: {}(stdout);{}(stderr)",lines.length(), errLines.length());
return new ProcessResults(lines.toString(), errLines.toString(), proc.exitValue());
}

private int runCmdAndPrintStreams(String[] cmd, long secondsToWait)
private ProcessResults runCmdAndPrintStreams(String[] cmd, long secondsToWait)
throws InterruptedException, IOException {
ProcessResults results = runCmd(cmd, secondsToWait);
LOG.info("Stdout from proc: " + results.stdout);
LOG.info("Stderr from proc: " + results.stderr);
return results.rc;
return results;
}

protected void printDockerEvents() {
try {
runCmdAndPrintStreams(new String[] { "docker", "events", "--since", "24h", "--until", "0s" }, 3);
} catch (Exception e) {
LOG.warn("A problem was encountered while attempting to retrieve Docker events (the system made an analytical"
+ " best effort to list the events to reveal the root cause). No further actions are necessary.", e);
}
}

private String[] buildRunCmd() {
Expand Down