Create dedicated hosts (#109)

Co-authored-by: Kamilla Aslami <kaslami@pivotal.io>
Co-authored-by: Jacob Barrett <jbarrett@pivotal.io>

infrastructure/scripts/aws/run_tests.sh

@@ -144,7 +144,7 @@ REPO=$(fixRepoName ${REPO})
 SSH_OPTIONS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ~/.geode-benchmarks/${TAG}-privkey.pem"
 HOSTS=`aws ec2 describe-instances --query 'Reservations[*].Instances[*].PrivateIpAddress' --filter "Name=tag:geode-benchmarks,Values=${TAG}" --output text`
 HOSTS=$(echo ${HOSTS} | tr ' ' ',')
-FIRST_INSTANCE=`aws ec2 describe-instances --query 'Reservations[*].Instances[*].PublicIpAddress' --filter "Name=tag:geode-benchmarks,Values=${TAG}" --output text | cut -f 1`
+FIRST_INSTANCE=`aws ec2 describe-instances --query 'Reservations[*].Instances[*].PublicIpAddress' --filter "Name=tag:geode-benchmarks,Values=${TAG}" --output text | cut -f 1 | head -n 1`
 
 echo "FIRST_INSTANCE=${FIRST_INSTANCE}"
 echo "HOSTS=${HOSTS}"

infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java

@@ -29,10 +29,14 @@
 import software.amazon.awssdk.services.ec2.model.DeleteLaunchTemplateRequest;
 import software.amazon.awssdk.services.ec2.model.DeletePlacementGroupRequest;
 import software.amazon.awssdk.services.ec2.model.DeleteSecurityGroupRequest;
+import software.amazon.awssdk.services.ec2.model.DescribeHostsRequest;
+import software.amazon.awssdk.services.ec2.model.DescribeHostsResponse;
 import software.amazon.awssdk.services.ec2.model.DescribeInstancesRequest;
 import software.amazon.awssdk.services.ec2.model.DescribeInstancesResponse;
 import software.amazon.awssdk.services.ec2.model.Filter;
+import software.amazon.awssdk.services.ec2.model.Host;
 import software.amazon.awssdk.services.ec2.model.Instance;
+import software.amazon.awssdk.services.ec2.model.ReleaseHostsRequest;
 import software.amazon.awssdk.services.ec2.model.TerminateInstancesRequest;
 
 import org.apache.geode.infrastructure.BenchmarkMetadata;
@@ -51,13 +55,29 @@ public static void main(String[] args) throws InterruptedException {
     }
 
     deleteInstances(benchmarkTag);
+    releaseHosts(benchmarkTag);
     deleteLaunchTemplate(benchmarkTag);
     deleteSecurityGroup(benchmarkTag);
     deletePlacementGroup(benchmarkTag);
     deleteKeyPair(benchmarkTag);
     deleteMetadata(benchmarkTag);
   }
 
+  private static void releaseHosts(String benchmarkTag) {
+    DescribeHostsResponse hosts = ec2.describeHosts(DescribeHostsRequest.builder()
+        .filter(Filter.builder()
+            .name("tag:" + BenchmarkMetadata.PREFIX)
+            .values(benchmarkTag)
+            .build())
+        .build());
+
+    List<String> hostIds = hosts.hosts().stream().map(Host::hostId).collect(Collectors.toList());
+
+    ec2.releaseHosts(ReleaseHostsRequest.builder().hostIds(hostIds).build());
+
+    System.out.println("Hosts for cluster '" + benchmarkTag + "' released.");
+  }
+
   private static void deleteKeyPair(String benchmarkTag) {
     try {
       System.out.println("Deleting cluster keypair: " + AwsBenchmarkMetadata.keyPair(benchmarkTag));

infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java

@@ -35,6 +35,8 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
 import software.amazon.awssdk.services.ec2.Ec2Client;
+import software.amazon.awssdk.services.ec2.model.AllocateHostsRequest;
+import software.amazon.awssdk.services.ec2.model.AllocateHostsResponse;
 import software.amazon.awssdk.services.ec2.model.AuthorizeSecurityGroupIngressRequest;
 import software.amazon.awssdk.services.ec2.model.CreateKeyPairRequest;
 import software.amazon.awssdk.services.ec2.model.CreateKeyPairResponse;
@@ -55,14 +57,15 @@
 import software.amazon.awssdk.services.ec2.model.Instance;
 import software.amazon.awssdk.services.ec2.model.LaunchTemplateBlockDeviceMappingRequest;
 import software.amazon.awssdk.services.ec2.model.LaunchTemplateEbsBlockDeviceRequest;
-import software.amazon.awssdk.services.ec2.model.LaunchTemplatePlacementRequest;
 import software.amazon.awssdk.services.ec2.model.LaunchTemplateSpecification;
+import software.amazon.awssdk.services.ec2.model.Placement;
 import software.amazon.awssdk.services.ec2.model.RequestLaunchTemplateData;
 import software.amazon.awssdk.services.ec2.model.ResourceType;
 import software.amazon.awssdk.services.ec2.model.RunInstancesRequest;
 import software.amazon.awssdk.services.ec2.model.RunInstancesResponse;
 import software.amazon.awssdk.services.ec2.model.Tag;
 import software.amazon.awssdk.services.ec2.model.TagSpecification;
+import software.amazon.awssdk.services.ec2.model.Tenancy;
 import software.amazon.awssdk.services.ec2.model.VolumeType;
 
 import org.apache.geode.infrastructure.BenchmarkMetadata;
@@ -93,7 +96,8 @@ public static void main(String[] args) throws IOException, InterruptedException
     authorizeSecurityGroup(benchmarkTag);
     createLaunchTemplate(benchmarkTag, newestImage);
 
-    List<String> instanceIds = launchInstances(benchmarkTag, tags, count);
+    List<String> hostIds = allocateHosts(tags, count);
+    List<String> instanceIds = launchInstances(benchmarkTag, tags, count, hostIds);
     DescribeInstancesResponse instances = waitForInstances(instanceIds);
     List<String> publicIps = getPublicIps(instances);
     createMetadata(benchmarkTag, publicIps);
@@ -113,27 +117,44 @@ private static void usage(String s) {
     throw new IllegalStateException(s);
   }
 
-  private static List<String> launchInstances(String benchmarkTag, List<Tag> tags,
-      int instanceCount)
-      throws InterruptedException {
-    // launch instances
-
-    RunInstancesResponse rir = ec2.runInstances(RunInstancesRequest.builder()
-        .launchTemplate(LaunchTemplateSpecification.builder()
-            .launchTemplateName(AwsBenchmarkMetadata.launchTemplate(benchmarkTag))
-            .build())
+  private static List<String> allocateHosts(List<Tag> tags, int count) {
+    AllocateHostsResponse hosts = ec2.allocateHosts(AllocateHostsRequest.builder()
+        .availabilityZone("us-west-2a")
+        .instanceType(AwsBenchmarkMetadata.instanceType().toString())
+        .quantity(count)
         .tagSpecifications(TagSpecification.builder()
             .tags(tags)
-            .resourceType(ResourceType.INSTANCE)
+            .resourceType(ResourceType.DEDICATED_HOST)
             .build())
-        .minCount(instanceCount)
-        .maxCount(instanceCount)
         .build());
 
-    List<String> instanceIds = rir.instances()
-        .stream()
-        .map(Instance::instanceId)
-        .collect(Collectors.toList());
+    return hosts.hostIds();
+  }
+
+  private static List<String> launchInstances(String launchTemplate, List<Tag> tags,
+      int instanceCount, List<String> hosts)
+      throws InterruptedException {
+    List<String> instanceIds = new ArrayList<>(instanceCount);
+    for (String host : hosts) {
+      // launch instances
+      RunInstancesResponse rir = ec2.runInstances(RunInstancesRequest.builder()
+          .launchTemplate(LaunchTemplateSpecification.builder()
+              .launchTemplateName(AwsBenchmarkMetadata.launchTemplate(launchTemplate))
+              .build())
+          .placement(Placement.builder()
+              .tenancy(Tenancy.HOST)
+              .hostId(host)
+              .build())
+          .tagSpecifications(TagSpecification.builder()
+              .tags(tags)
+              .resourceType(ResourceType.INSTANCE)
+              .build())
+          .minCount(1)
+          .maxCount(1)
+          .build());
+
+      instanceIds.add(rir.instances().get(0).instanceId());
+    }
 
     return instanceIds;
   }
@@ -222,10 +243,6 @@ private static void createLaunchTemplate(String benchmarkTag, Image newestImage)
             .launchTemplateData(RequestLaunchTemplateData.builder()
                 .imageId(newestImage.imageId())
                 .instanceType(AwsBenchmarkMetadata.instanceType())
-                .placement(LaunchTemplatePlacementRequest.builder()
-                    .groupName(AwsBenchmarkMetadata.placementGroup(benchmarkTag))
-                    .tenancy(AwsBenchmarkMetadata.tenancy())
-                    .build())
                 .keyName(AwsBenchmarkMetadata.keyPair(benchmarkTag))
                 .securityGroups(securityGroupList)
                 .blockDeviceMappings(LaunchTemplateBlockDeviceMappingRequest.builder()