Skip to content
Permalink
Browse files
GEODE-6444: Store GC logs in files (#56)
* Store the GC logs in a file.
	* Print the GC logs in the System.out / System.err in parallel while the benchmark is being executed in parallel.
	* Using a thread pool for execution of runAsync and not depend on fork join pools
        * Disabling the dstat probe

Co-authored-by: Dan Smith <dsmith@pivotal.io>
Co-authored-by: nabarunnag <nag@cs.wisc.edu>
Co-authored-by: Jacob Barrett <jbarrett@pivotal.io>
  • Loading branch information
4 people committed Feb 25, 2019
1 parent 7a90788 commit c477ca02a9a4685c2008d9b9ea98d2e500522592
Show file tree
Hide file tree
Showing 7 changed files with 34 additions and 27 deletions.
@@ -61,8 +61,6 @@ task benchmark(type: Test) {
exceptionFormat = 'full'
}

exclude "**/*Function*Benchmark.class"
exclude "**/PartitionedIndexQueryBenchmark.class"
forkEvery 1

systemProperty 'TEST_HOSTS', project.findProperty('hosts')
@@ -31,6 +31,8 @@
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Function;
import java.util.stream.Collectors;

@@ -52,6 +54,7 @@ public class SshInfrastructure implements Infrastructure {
private final String user;
private final int port;
public static final Config CONFIG = new DefaultConfig();
private ExecutorService streamReaderThreadPool = Executors.newCachedThreadPool();

public SshInfrastructure(Collection<String> hosts, String user) {
this(hosts, user, 22);
@@ -88,18 +91,36 @@ public int onNode(Node node, String[] shellCommand)
try (Session session = client.startSession()) {
logger.info("Executing " + script + " on " + node.getAddress());
final Session.Command cmd = session.exec(script);
cmd.join();
copyStream(cmd.getInputStream(), System.out);
copyStream(cmd.getErrorStream(), System.err);
CompletableFuture<Void> copyStdout =
copyStreamAsynchronously(cmd.getInputStream(), System.out);
CompletableFuture<Void> copyStdErr =
copyStreamAsynchronously(cmd.getErrorStream(), System.err);

cmd.join();
copyStdout.join();
copyStdErr.join();
return cmd.getExitStatus();
}
}
}

private void copyStream(InputStream inputStream, PrintStream out) throws IOException {
org.apache.commons.io.IOUtils.copy(inputStream, out);
private CompletableFuture<Void> copyStreamAsynchronously(InputStream inputStream,
PrintStream out) {
return CompletableFuture.runAsync(() -> {
try {
org.apache.commons.io.IOUtils.copy(inputStream, out);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}, streamReaderThreadPool);
}

private void copyStream(InputStream inputStream, PrintStream out) {
try {
org.apache.commons.io.IOUtils.copy(inputStream, out);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

@Override
@@ -56,6 +56,8 @@ CompletableFuture<Void> launchWorker(Infrastructure infra, int rmiPort,
public void run() {

try {
infra.onNode(jvmConfig.getNode(), new String[] {"rm", "-rf", jvmConfig.getOutputDir()});
infra.onNode(jvmConfig.getNode(), new String[] {"mkdir", "-p", jvmConfig.getOutputDir()});
int result = infra.onNode(jvmConfig.getNode(), shellCommand);
if (result != 0) {
logger.error("ChildJVM exited with error code " + result);
@@ -83,6 +85,7 @@ String[] buildCommand(String rmiHost, int rmiPort, JVMMapping jvmConfig) {
command.add("-D" + RemoteJVMFactory.RMI_PORT_PROPERTY + "=" + rmiPort);
command.add("-D" + RemoteJVMFactory.JVM_ID + "=" + jvmConfig.getId());
command.add("-D" + RemoteJVMFactory.OUTPUT_DIR + "=" + jvmConfig.getOutputDir());
command.add("-Xloggc:" + jvmConfig.getOutputDir() + "/gc.log");
command.addAll(jvmConfig.getJvmArgs());
command.add(ChildJVM.class.getName());

@@ -21,8 +21,6 @@
import java.io.PrintStream;
import java.rmi.RemoteException;

import org.apache.commons.io.FileUtils;

import org.apache.geode.perftest.jdk.RMI;
import org.apache.geode.perftest.jdk.SystemInterface;
import org.apache.geode.perftest.jvms.RemoteJVMFactory;
@@ -62,9 +60,6 @@ void run() {
}

File outputDir = new File(OUTPUT_DIR);
// Clean up the output directory before the test runs
FileUtils.deleteQuietly(outputDir);
outputDir.mkdirs();
try (PrintStream out = new PrintStream(new File(outputDir, "system.log"))) {
system.setOut(out);
system.setErr(out);
@@ -25,6 +25,8 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.apache.geode.perftest.Task;
@@ -39,6 +41,7 @@ public class Controller extends UnicastRemoteObject implements ControllerRemote
private final Map<Integer, WorkerRemote> workers = new ConcurrentHashMap<>();
private final CountDownLatch workersStarted;
private volatile boolean isClosed;
private ExecutorService workerExecutionPool = Executors.newCachedThreadPool();


Controller(int numWorkers, Registry registry, SharedContext context) throws RemoteException {
@@ -85,6 +88,6 @@ public CompletableFuture<Void> onWorker(int id, Task task) {
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}, workerExecutionPool);
}
}
@@ -27,7 +27,6 @@
import org.yardstickframework.impl.BenchmarkLoader;
import org.yardstickframework.impl.BenchmarkProbeSet;
import org.yardstickframework.impl.BenchmarkRunner;
import org.yardstickframework.probes.DStatProbe;
import org.yardstickframework.probes.PercentileProbe;
import org.yardstickframework.probes.ThroughputLatencyProbe;
import org.yardstickframework.probes.VmStatProbe;
@@ -95,7 +94,7 @@ public String defaultDescription() {
Collection<BenchmarkProbe> probes =
Arrays.asList(new HdrHistogramProbe(new HdrHistogramWriter(context.getOutputDir())),
new ThroughputLatencyProbe(),
new PercentileProbe(), new DStatProbe(), new VmStatProbe(),
new PercentileProbe(), new VmStatProbe(),
testDoneProbe);
BenchmarkLoader loader = new BenchmarkLoader();
loader.initialize(cfg);
@@ -20,7 +20,6 @@
import static org.apache.geode.perftest.jvms.RemoteJVMFactory.OUTPUT_DIR;
import static org.apache.geode.perftest.jvms.RemoteJVMFactory.RMI_HOST;
import static org.apache.geode.perftest.jvms.RemoteJVMFactory.RMI_PORT_PROPERTY;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
@@ -83,15 +82,4 @@ public void childRetriesUntilControllerExits() throws RemoteException {

verify(controller, times(3)).ping();
}

@Test
public void childCleansOutputDir() throws IOException {
File expectedFile = new File(folder, "somefile.txt");
expectedFile.createNewFile();

jvm.run();

assertFalse(expectedFile.exists());
}

}

0 comments on commit c477ca0

Please sign in to comment.