Skip to content
Permalink
Browse files
Refactor metadata production and management. (#57)
* Tests now output metadata if the metadata file doesn't exist.
* benchmark run metadata.json is actually json.
* Create configuration directory to store cluster information.
* Move SSH private key for cluster to configuration directory.
* Create metadata for cluster, currently including an instance ID and
  the list of IPs in the cluster.
* Delete metadata when the cluster is destroyed.
  • Loading branch information
smgoller committed Feb 27, 2019
1 parent dec1d89 commit 7d6dfc6b44b6d0822ca8a39e3d4f2fe348fc6e78
Show file tree
Hide file tree
Showing 11 changed files with 184 additions and 25 deletions.
@@ -28,3 +28,4 @@ awaitility.version = 3.0.0
sshd-core.version = 2.1.0
assertj-core.version = 3.11.1
software-amazon-awssdk.version = 2.1.4
JSON.version = 20180813
@@ -41,6 +41,7 @@ dependencies {
compile(group: 'commons-io', name: 'commons-io', version: project.'commons-io.version')
compile(group: 'org.yardstickframework', name: 'yardstick', version: project.'yardstick.version')
compile(group: 'org.hdrhistogram', name: 'HdrHistogram', version: project.'HdrHistogram.version')
compile(group: 'org.json', name: 'json', version: project.'JSON.version')
testCompile(group: 'org.mockito', name: 'mockito-all', version: project.'mockito-all.version')
testCompile(group: 'org.awaitility', name: 'awaitility', version: project.'awaitility.version')
testCompile(group: 'org.slf4j', name: 'slf4j-simple', version: project.'slf4j-simple.version')
@@ -22,6 +22,7 @@
import java.util.List;
import java.util.Map;

import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -76,9 +77,15 @@ protected void runTest(TestConfig config, String testName)
FileWriter metadataWriter = new FileWriter(metadataOutput.getAbsoluteFile(), true);

String[] metadataEntries = metadata.split(",");
JSONObject JSONmetadata = new JSONObject();

for (String data : metadataEntries) {
metadataWriter.write(data + "\n");
String[] kv = data.split(":");
if (kv.length == 2) {
JSONmetadata.put(kv[0], kv[1]);
}
}
metadataWriter.write(JSONmetadata.toString());
metadataWriter.flush();
}

@@ -26,14 +26,17 @@
public class HdrHistogramWriter implements Consumer<Histogram> {

public static final String FILE_NAME = "latency.hlog";
public static final String FILE_NAME_CSV = "latency_csv";
public static final String FILE_NAME_HDR = "latency_hdr";

private final File outputFile;
private final File outputHDRFile;
private final File outputCSVFile;

public HdrHistogramWriter(File outputDir) {
this.outputFile = new File(outputDir, FILE_NAME);
this.outputHDRFile = new File(outputDir, FILE_NAME_HDR);
this.outputCSVFile = new File(outputDir, FILE_NAME_CSV);
}

@Override
@@ -51,6 +54,10 @@ public void accept(Histogram histogram) {
new HistogramLogProcessor(new String[] {"-i", outputFile.getAbsolutePath(), "-o",
outputHDRFile.getAbsolutePath()});
histogramLogProcessor.run();
HistogramLogProcessor histogramLogProcessorCSV =
new HistogramLogProcessor(new String[] {"-csv", "-i", outputFile.getAbsolutePath(), "-o",
outputCSVFile.getAbsolutePath()});
histogramLogProcessorCSV.run();
} catch (FileNotFoundException e) {
throw new UncheckedIOException(e);
}
@@ -28,6 +28,7 @@ repositories {
}

dependencies {
implementation(group: 'org.json', name: 'json', version: project.'JSON.version')
implementation 'software.amazon.awssdk:ec2'
implementation(group: 'com.hierynomus', name: 'sshj', version: project.'sshj.version')
runtime(group: 'org.slf4j', name: 'slf4j-simple', version: project.'slf4j-simple.version')
@@ -21,7 +21,6 @@
*/
public class BenchmarkMetadata {
public static String PREFIX = "geode-benchmarks";
public static String SSH_DIRECTORY = ".ssh/geode-benchmarks";

public static String benchmarkPrefix(String tag) {
return PREFIX + "-" + tag;
@@ -31,11 +30,16 @@ public static String benchmarkString(String tag, String suffix) {
return benchmarkPrefix(tag) + "-" + suffix;
}

public static String benchmarkKeyFileDirectory() {
return System.getProperty("user.home") + "/" + SSH_DIRECTORY;
public static String benchmarkConfigDirectory() {
return System.getProperty("user.home") + "/." + PREFIX;
}

public static String benchmarkKeyFileName(String tag) {
return benchmarkKeyFileDirectory() + "/" + tag + ".pem";

public static String benchmarkPrivateKeyFileName(String tag) {
return benchmarkConfigDirectory() + "/" + tag + "-privkey.pem";
}

public static String benchmarkMetadataFileName(String tag) {
return benchmarkConfigDirectory() + "/" + tag + "-metadata.json";
}
}
@@ -48,7 +48,11 @@ public static String keyPair(String tag) {
}

public static String keyPairFileName(String tag) {
return BenchmarkMetadata.benchmarkKeyFileName(tag);
return BenchmarkMetadata.benchmarkPrivateKeyFileName(tag);
}

public static String metadataFileName(String tag) {
return BenchmarkMetadata.benchmarkMetadataFileName(tag);
}

public static InstanceType instanceType() {
@@ -55,6 +55,7 @@ public static void main(String[] args) throws InterruptedException {
deleteSecurityGroup(benchmarkTag);
deletePlacementGroup(benchmarkTag);
deleteKeyPair(benchmarkTag);
deleteMetadata(benchmarkTag);
}

private static void deleteKeyPair(String benchmarkTag) {
@@ -71,6 +72,16 @@ private static void deleteKeyPair(String benchmarkTag) {
}
}

private static void deleteMetadata(String benchmarkTag) {
try {
Files.deleteIfExists(Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag)));
System.out.println("Metadata for cluster '" + benchmarkTag + "' deleted.");
} catch (Exception e) {
System.out.println("We got an exception while deleting the Key pair");
System.out.println("Exception message: " + e);
}
}

private static void deleteInstances(String benchmarkTag) throws InterruptedException {
// delete instances
try {
@@ -21,6 +21,7 @@
import java.io.UncheckedIOException;
import java.net.ConnectException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.Collections;

@@ -39,12 +40,11 @@ public class KeyInstaller {
private final String user;
private final Path privateKey;

public KeyInstaller(String user, Path privateKey) {
this.user = user;
this.privateKey = privateKey;
public KeyInstaller(String benchmarkTag) {
this.user = AwsBenchmarkMetadata.USER;
this.privateKey = Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag));
}


public void installPrivateKey(Collection<String> hosts) {
hosts.forEach(this::installKey);
}
@@ -29,8 +29,11 @@
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
import java.util.stream.Collectors;

import org.json.JSONArray;
import org.json.JSONObject;
import software.amazon.awssdk.services.ec2.Ec2Client;
import software.amazon.awssdk.services.ec2.model.AuthorizeSecurityGroupIngressRequest;
import software.amazon.awssdk.services.ec2.model.CreateKeyPairRequest;
@@ -83,12 +86,20 @@ public static void main(String[] args) throws IOException, InterruptedException

List<String> instanceIds = launchInstances(benchmarkTag, tags, count);
DescribeInstancesResponse instances = waitForInstances(instanceIds);

List<String> publicIps = installPrivateKey(benchmarkTag, instances);
List<String> publicIps = getPublicIps(instances);
createMetadata(benchmarkTag, publicIps);
installPrivateKey(benchmarkTag, publicIps);
installMetadata(benchmarkTag, publicIps);

System.out.println("Instances successfully launched! Public IPs: " + publicIps);
}

private static List<String> getPublicIps(DescribeInstancesResponse describeInstancesResponse) {
return describeInstancesResponse.reservations().stream()
.flatMap(reservation -> reservation.instances().stream())
.map(Instance::publicIpAddress).collect(Collectors.toList());
}

private static void usage(String s) {
throw new IllegalStateException(s);
}
@@ -132,19 +143,16 @@ private static DescribeInstancesResponse waitForInstances(List<String> instanceI
return describeInstancesResponse;
}

private static List<String> installPrivateKey(String benchmarkTag,
DescribeInstancesResponse describeInstancesResponse) {
List<String> publicIps =
describeInstancesResponse.reservations().stream()
.flatMap(reservation -> reservation.instances().stream())
.map(Instance::publicIpAddress).collect(Collectors.toList());

new KeyInstaller(AwsBenchmarkMetadata.USER,
Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag))).installPrivateKey(publicIps);

private static void installPrivateKey(String benchmarkTag,
List<String> publicIps) {
new KeyInstaller(benchmarkTag).installPrivateKey(publicIps);
System.out.println("Private key installed on all instances for passwordless ssh");
}

return publicIps;
private static void installMetadata(String benchmarkTag,
List<String> publicIps) {
new MetadataInstaller(benchmarkTag).installMetadata(publicIps);
System.out.println("Instance ID information installed on all instances");
}

private static long instanceCount(DescribeInstancesResponse describeInstancesResponse) {
@@ -164,14 +172,36 @@ private static DescribeInstancesResponse describeInstances(List<String> instance
}

private static void createKeyPair(String benchmarkTag) throws IOException {
Path configDirectory = Paths.get(BenchmarkMetadata.benchmarkConfigDirectory());
CreateKeyPairResponse ckpr = ec2.createKeyPair(
CreateKeyPairRequest.builder().keyName(AwsBenchmarkMetadata.keyPair(benchmarkTag)).build());
Files.createDirectories(Paths.get(BenchmarkMetadata.benchmarkKeyFileDirectory()));

if (!configDirectory.toFile().exists()) {
Files.createDirectories(Paths.get(BenchmarkMetadata.benchmarkConfigDirectory()));
}
Path privateKey = Files.write(Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag)),
ckpr.keyMaterial().getBytes());
Files.setPosixFilePermissions(privateKey, PosixFilePermissions.fromString("rw-------"));
}

private static void createMetadata(String benchmarkTag, List<String> publicIps)
throws IOException {
UUID instanceId = UUID.randomUUID();
JSONObject metadataJSON = new JSONObject();

metadataJSON.put("instanceId", instanceId.toString());
metadataJSON.put("publicIps", new JSONArray(publicIps));
Path configDirectory = Paths.get(BenchmarkMetadata.benchmarkConfigDirectory());

if (!configDirectory.toFile().exists()) {
Files.createDirectories(Paths.get(BenchmarkMetadata.benchmarkConfigDirectory()));
}

Path metadata = Files.write(Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag)),
metadataJSON.toString().getBytes());
Files.setPosixFilePermissions(metadata, PosixFilePermissions.fromString("rw-------"));
}

private static void createLaunchTemplate(String benchmarkTag, Image newestImage) {
ArrayList<String> securityGroupList = new ArrayList<>();
securityGroupList.add(AwsBenchmarkMetadata.securityGroup(benchmarkTag));
@@ -233,6 +263,8 @@ private static List<Tag> getTags(String benchmarkTag) {
List<Tag> tags = new ArrayList<>();
tags.add(Tag.builder().key("purpose").value(BenchmarkMetadata.PREFIX).build());
tags.add(Tag.builder().key(BenchmarkMetadata.PREFIX).value(benchmarkTag).build());
tags.add(Tag.builder().key(BenchmarkMetadata.benchmarkString(benchmarkTag, "instanceId"))
.value(UUID.randomUUID().toString()).build());
return tags;
}

@@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.geode.infrastructure.aws;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.ConnectException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.Collections;

import net.schmizz.sshj.Config;
import net.schmizz.sshj.DefaultConfig;
import net.schmizz.sshj.SSHClient;
import net.schmizz.sshj.sftp.FileAttributes;
import net.schmizz.sshj.sftp.SFTPClient;
import net.schmizz.sshj.transport.verification.PromiscuousVerifier;
import net.schmizz.sshj.xfer.FilePermission;
import net.schmizz.sshj.xfer.FileSystemFile;


public class MetadataInstaller {
public static final Config CONFIG = new DefaultConfig();
private static final int RETRIES = 30;
private final String user;
private final Path metadata;
private final Path privateKey;

public MetadataInstaller(String benchmarkTag) {
this.user = AwsBenchmarkMetadata.USER;
this.privateKey = Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag));
this.metadata = Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag));
}

public void installMetadata(Collection<String> hosts) {
hosts.forEach(this::installMetadata);
}

private void installMetadata(String host) {
try (SSHClient client = new SSHClient(CONFIG)) {
client.addHostKeyVerifier(new PromiscuousVerifier());
connect(host, client);
client.authPublickey(user, privateKey.toFile().getAbsolutePath());
SFTPClient sftpClient = client.newSFTPClient();
String dest = "/home/" + user + "/geode-benchmarks-metadata.json";

sftpClient.put(new FileSystemFile(metadata.toFile()), dest);
sftpClient.setattr(dest, new FileAttributes.Builder()
.withPermissions(Collections.singleton(FilePermission.USR_R)).build());
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}

private void connect(String host, SSHClient client) throws IOException, InterruptedException {

int i = 0;
while (true) {
try {
i++;
client.connect(host);
return;
} catch (ConnectException e) {
if (i >= RETRIES) {
throw e;
}

System.out.println("Failed to connect, retrying...");
Thread.sleep(AwsBenchmarkMetadata.POLL_INTERVAL);
}
}
}
}

0 comments on commit 7d6dfc6

Please sign in to comment.