diff --git a/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/CloudFormationHelper.java b/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/CloudFormationHelper.java index 12f48a586bd..af7316f3975 100644 --- a/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/CloudFormationHelper.java +++ b/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/CloudFormationHelper.java @@ -10,11 +10,7 @@ import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.services.cloudformation.CloudFormationAsyncClient; -import software.amazon.awssdk.services.cloudformation.model.Capability; -import software.amazon.awssdk.services.cloudformation.model.CloudFormationException; -import software.amazon.awssdk.services.cloudformation.model.DescribeStacksRequest; -import software.amazon.awssdk.services.cloudformation.model.DescribeStacksResponse; -import software.amazon.awssdk.services.cloudformation.model.Output; +import software.amazon.awssdk.services.cloudformation.model.*; import software.amazon.awssdk.services.cloudformation.model.Stack; import software.amazon.awssdk.services.cloudformation.waiters.CloudFormationAsyncWaiter; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -26,9 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -39,150 +33,187 @@ public class CloudFormationHelper { private static CloudFormationAsyncClient cloudFormationClient; public static void main(String[] args) { + if (args.length < 1) { + logger.error("Usage: CloudFormationHelper "); + return; + } emptyS3Bucket(args[0]); } private static CloudFormationAsyncClient getCloudFormationClient() { if (cloudFormationClient == null) { SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder() - .maxConcurrency(100) - .connectionTimeout(Duration.ofSeconds(60)) - .readTimeout(Duration.ofSeconds(60)) - .writeTimeout(Duration.ofSeconds(60)) - .build(); + .maxConcurrency(100) + .connectionTimeout(Duration.ofSeconds(60)) + .readTimeout(Duration.ofSeconds(60)) + .writeTimeout(Duration.ofSeconds(60)) + .build(); ClientOverrideConfiguration overrideConfig = ClientOverrideConfiguration.builder() - .apiCallTimeout(Duration.ofMinutes(2)) - .apiCallAttemptTimeout(Duration.ofSeconds(90)) - .retryStrategy(RetryMode.STANDARD) - .build(); + .apiCallTimeout(Duration.ofMinutes(2)) + .apiCallAttemptTimeout(Duration.ofSeconds(90)) + .retryStrategy(RetryMode.STANDARD) + .build(); cloudFormationClient = CloudFormationAsyncClient.builder() - .httpClient(httpClient) - .overrideConfiguration(overrideConfig) - .build(); + .httpClient(httpClient) + .overrideConfiguration(overrideConfig) + .build(); } return cloudFormationClient; } public static void deployCloudFormationStack(String stackName) { - String templateBody; + logger.info("Deploying CloudFormation stack: {}", stackName); boolean doesExist = describeStack(stackName); if (!doesExist) { + String templateBody; try { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - Path filePath = Paths.get(classLoader.getResource(CFN_TEMPLATE).toURI()); + Path filePath = Paths.get(Objects.requireNonNull(classLoader.getResource(CFN_TEMPLATE)).toURI()); templateBody = Files.readString(filePath); } catch (IOException | URISyntaxException e) { + logger.error("Failed to read CloudFormation template", e); throw new RuntimeException(e); } getCloudFormationClient().createStack(b -> b.stackName(stackName) - .templateBody(templateBody) - .capabilities(Capability.CAPABILITY_IAM)) - .whenComplete((csr, t) -> { - if (csr != null) { - System.out.println("Stack creation requested, ARN is " + csr.stackId()); + .templateBody(templateBody) + .capabilities(Capability.CAPABILITY_IAM)) + .whenComplete((csr, t) -> { + if (t != null) { + logger.error("Error creating stack {}", stackName, t); + throw new RuntimeException("Stack creation failed", t); + } + + logger.info("Stack creation requested. ARN: {}", csr.stackId()); + try (CloudFormationAsyncWaiter waiter = getCloudFormationClient().waiter()) { waiter.waitUntilStackCreateComplete(request -> request.stackName(stackName)) - .whenComplete((dsr, th) -> { - if (th != null) { - System.out.println("Error waiting for stack creation: " + th.getMessage()); - } else { - dsr.matched().response().orElseThrow(() -> new RuntimeException("Failed to deploy")); - System.out.println("Stack created successfully"); - } - }).join(); + .whenComplete((dsr, th) -> { + if (th != null) { + logger.error("Error waiting for stack creation: {}", stackName, th); + } else { + dsr.matched().response() + .orElseThrow(() -> new RuntimeException("Stack creation failed for " + stackName)); + logger.info("Stack {} created successfully.", stackName); + + // Print outputs immediately + getStackOutputsAsync(stackName).whenComplete((outputs, outEx) -> { + if (outEx != null) { + logger.error("Failed to fetch stack outputs", outEx); + } else { + logger.info("Stack Outputs for {}:", stackName); + outputs.forEach((k, v) -> logger.info(" {} = {}", k, v)); + } + }).join(); + } + }).join(); } - } else { - System.out.format("Error creating stack: " + t.getMessage(), t); - throw new RuntimeException(t.getCause().getMessage(), t); - } - }).join(); + }).join(); } else { - logger.info("{} stack already exists", stackName); + logger.info("Stack {} already exists, skipping creation.", stackName); } } // Check to see if the Stack exists before deploying it public static Boolean describeStack(String stackName) { try { - CompletableFuture future = getCloudFormationClient().describeStacks(); - DescribeStacksResponse stacksResponse = (DescribeStacksResponse) future.join(); - List stacks = stacksResponse.stacks(); - for (Stack myStack : stacks) { - if (myStack.stackName().compareTo(stackName) == 0) { + CompletableFuture future = getCloudFormationClient().describeStacks(); + DescribeStacksResponse stacksResponse = future.join(); + for (Stack myStack : stacksResponse.stacks()) { + if (myStack.stackName().equals(stackName)) { + logger.info("Stack {} exists already.", stackName); return true; } } } catch (CloudFormationException e) { - System.err.println(e.getMessage()); + logger.error("Error describing stack {}", stackName, e); } return false; } public static void destroyCloudFormationStack(String stackName) { + logger.info("Deleting CloudFormation stack: {}", stackName); getCloudFormationClient().deleteStack(b -> b.stackName(stackName)) - .whenComplete((dsr, t) -> { - if (dsr != null) { - System.out.println("Delete stack requested ...."); + .whenComplete((dsr, t) -> { + if (t != null) { + logger.error("Error deleting stack {}", stackName, t); + throw new RuntimeException("Delete failed", t); + } + + logger.info("Delete stack requested: {}", stackName); try (CloudFormationAsyncWaiter waiter = getCloudFormationClient().waiter()) { waiter.waitUntilStackDeleteComplete(request -> request.stackName(stackName)) - .whenComplete((waiterResponse, throwable) -> - System.out.println("Stack deleted successfully.")) - .join(); + .whenComplete((waiterResponse, throwable) -> { + if (throwable != null) { + logger.error("Error waiting for stack deletion {}", stackName, throwable); + } else { + logger.info("Stack {} deleted successfully.", stackName); + } + }).join(); } - } else { - System.out.format("Error deleting stack: " + t.getMessage(), t); - throw new RuntimeException(t.getCause().getMessage(), t); - } - }).join(); + }).join(); } public static CompletableFuture> getStackOutputsAsync(String stackName) { - CloudFormationAsyncClient cloudFormationAsyncClient = getCloudFormationClient(); + logger.info("Fetching stack outputs for {}", stackName); DescribeStacksRequest describeStacksRequest = DescribeStacksRequest.builder() - .stackName(stackName) - .build(); - - return cloudFormationAsyncClient.describeStacks(describeStacksRequest) - .handle((describeStacksResponse, throwable) -> { - if (throwable != null) { - throw new RuntimeException("Failed to get stack outputs for: " + stackName, throwable); - } + .stackName(stackName) + .build(); - // Process the result - if (describeStacksResponse.stacks().isEmpty()) { - throw new RuntimeException("Stack not found: " + stackName); - } + return getCloudFormationClient().describeStacks(describeStacksRequest) + .handle((describeStacksResponse, throwable) -> { + if (throwable != null) { + logger.error("Failed to get stack outputs for {}", stackName, throwable); + throw new RuntimeException("Failed to get stack outputs for: " + stackName, throwable); + } - Stack stack = describeStacksResponse.stacks().get(0); - Map outputs = new HashMap<>(); - for (Output output : stack.outputs()) { - outputs.put(output.outputKey(), output.outputValue()); - } + if (describeStacksResponse.stacks().isEmpty()) { + throw new RuntimeException("Stack not found: " + stackName); + } - return outputs; - }); + Stack stack = describeStacksResponse.stacks().get(0); + Map outputs = new HashMap<>(); + for (Output output : stack.outputs()) { + outputs.put(output.outputKey(), output.outputValue()); + } + return outputs; + }); } public static void emptyS3Bucket(String bucketName) { + logger.info("Emptying S3 bucket: {}", bucketName); S3AsyncClient s3Client = S3AsyncClient.builder().build(); s3Client.listObjectsV2(req -> req.bucket(bucketName)) - .thenCompose(response -> { - List> deleteFutures = response.contents().stream() - .map(s3Object -> s3Client.deleteObject(req -> req - .bucket(bucketName) - .key(s3Object.key()))) - .collect(Collectors.toList()); + .thenCompose(response -> { + if (response.contents().isEmpty()) { + logger.info("Bucket {} is already empty.", bucketName); + return CompletableFuture.completedFuture(null); + } - return CompletableFuture.allOf(deleteFutures.toArray(new CompletableFuture[0])); - }) - .join(); + List> deleteFutures = response.contents().stream() + .map(s3Object -> { + logger.info("Deleting object: {}", s3Object.key()); + return s3Client.deleteObject(req -> req + .bucket(bucketName) + .key(s3Object.key())); + }) + .collect(Collectors.toList()); + + return CompletableFuture.allOf(deleteFutures.toArray(new CompletableFuture[0])); + }) + .whenComplete((res, ex) -> { + if (ex != null) { + logger.error("Failed to empty bucket {}", bucketName, ex); + } else { + logger.info("Bucket {} emptied successfully.", bucketName); + } + }) + .join(); s3Client.close(); } } - diff --git a/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/EntityResScenario.java b/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/EntityResScenario.java index 75f7dfc26f4..ef5e9e8002f 100644 --- a/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/EntityResScenario.java +++ b/javav2/example_code/entityresolution/src/main/java/com/example/entity/scenario/EntityResScenario.java @@ -23,7 +23,7 @@ public class EntityResScenario { private static final Logger logger = LoggerFactory.getLogger(EntityResScenario.class); public static final String DASHES = new String(new char[80]).replace("\0", "-"); - private static final String STACK_NAME = "EntityResolutionCdkStack"; + private static final String STACK_NAME = "EntityResolutionCdkStack2"; private static final String ENTITY_RESOLUTION_ROLE_ARN_KEY = "EntityResolutionRoleArn"; private static final String GLUE_DATA_BUCKET_NAME_KEY = "GlueDataBucketName"; private static final String JSON_GLUE_TABLE_ARN_KEY = "JsonErGlueTableArn"; diff --git a/javav2/example_code/entityresolution/src/main/resources/template.yaml b/javav2/example_code/entityresolution/src/main/resources/template.yaml index f0395929fa7..851b65c8e68 100644 --- a/javav2/example_code/entityresolution/src/main/resources/template.yaml +++ b/javav2/example_code/entityresolution/src/main/resources/template.yaml @@ -1,28 +1,30 @@ Resources: - ErBucket6EA35F9D: + ERBucketF684533D: Type: AWS::S3::Bucket Properties: - BucketName: erbucketf684533d2680435fa99d24b1bdaf5179 + BucketName: !Sub er-bucket-${AWS::AccountId}-${AWS::Region} + Tags: + - Key: Name + Value: ERBucket UpdateReplacePolicy: Delete DeletionPolicy: Delete Metadata: aws:cdk:path: EntityResolutionCdkStack/ErBucket/Resource + GlueDatabase: Type: AWS::Glue::Database Properties: - CatalogId: - Ref: AWS::AccountId + CatalogId: !Ref AWS::AccountId DatabaseInput: Name: entity_resolution_db Metadata: aws:cdk:path: EntityResolutionCdkStack/GlueDatabase + jsongluetable: Type: AWS::Glue::Table Properties: - CatalogId: - Ref: AWS::AccountId - DatabaseName: - Ref: GlueDatabase + CatalogId: !Ref AWS::AccountId + DatabaseName: !Ref GlueDatabase TableInput: Name: jsongluetable StorageDescriptor: @@ -34,12 +36,7 @@ Resources: - Name: email Type: string InputFormat: org.apache.hadoop.mapred.TextInputFormat - Location: - Fn::Join: - - "" - - - s3:// - - Ref: ErBucket6EA35F9D - - /jsonData/ + Location: !Sub s3://${ERBucketF684533D}/jsonData/ OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat SerdeInfo: Parameters: @@ -50,13 +47,12 @@ Resources: - GlueDatabase Metadata: aws:cdk:path: EntityResolutionCdkStack/jsongluetable + csvgluetable: Type: AWS::Glue::Table Properties: - CatalogId: - Ref: AWS::AccountId - DatabaseName: - Ref: GlueDatabase + CatalogId: !Ref AWS::AccountId + DatabaseName: !Ref GlueDatabase TableInput: Name: csvgluetable StorageDescriptor: @@ -70,12 +66,7 @@ Resources: - Name: phone Type: string InputFormat: org.apache.hadoop.mapred.TextInputFormat - Location: - Fn::Join: - - "" - - - s3:// - - Ref: ErBucket6EA35F9D - - /csvData/ + Location: !Sub s3://${ERBucketF684533D}/csvData/ OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat SerdeInfo: Parameters: @@ -86,6 +77,7 @@ Resources: - GlueDatabase Metadata: aws:cdk:path: EntityResolutionCdkStack/csvgluetable + EntityResolutionRoleB51A51D3: Type: AWS::IAM::Role Properties: @@ -97,28 +89,13 @@ Resources: Service: entityresolution.amazonaws.com Version: "2012-10-17" ManagedPolicyArns: - - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - :iam::aws:policy/AmazonS3FullAccess - - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - :iam::aws:policy/AWSEntityResolutionConsoleFullAccess - - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - :iam::aws:policy/AWSGlueConsoleFullAccess - - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - :iam::aws:policy/service-role/AWSGlueServiceRole + - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonS3FullAccess + - !Sub arn:${AWS::Partition}:iam::aws:policy/AWSEntityResolutionConsoleFullAccess + - !Sub arn:${AWS::Partition}:iam::aws:policy/AWSGlueConsoleFullAccess + - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSGlueServiceRole Metadata: aws:cdk:path: EntityResolutionCdkStack/EntityResolutionRole/Resource + EntityResolutionRoleDefaultPolicy586C8066: Type: AWS::IAM::Policy Properties: @@ -135,6 +112,7 @@ Resources: - Ref: EntityResolutionRoleB51A51D3 Metadata: aws:cdk:path: EntityResolutionCdkStack/EntityResolutionRole/DefaultPolicy/Resource + CDKMetadata: Type: AWS::CDK::Metadata Properties: @@ -142,122 +120,57 @@ Resources: Metadata: aws:cdk:path: EntityResolutionCdkStack/CDKMetadata/Default Condition: CDKMetadataAvailable + Outputs: EntityResolutionRoleArn: Description: The ARN of the EntityResolution Role - Value: - Fn::GetAtt: - - EntityResolutionRoleB51A51D3 - - Arn + Value: !GetAtt EntityResolutionRoleB51A51D3.Arn + JsonErGlueTableArn: Description: The ARN of the Json Glue Table - Value: - Fn::Join: - - "" - - - "arn:aws:glue:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :table/ - - Ref: GlueDatabase - - /jsongluetable + Value: !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/${GlueDatabase}/jsongluetable + CsvErGlueTableArn: Description: The ARN of the CSV Glue Table - Value: - Fn::Join: - - "" - - - "arn:aws:glue:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :table/ - - Ref: GlueDatabase - - /csvgluetable + Value: !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/${GlueDatabase}/csvgluetable + GlueDataBucketName: Description: The name of the Glue Data Bucket - Value: - Ref: ErBucket6EA35F9D + Value: !Ref ERBucketF684533D + Conditions: CDKMetadataAvailable: Fn::Or: - Fn::Or: - - Fn::Equals: - - Ref: AWS::Region - - af-south-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-east-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-northeast-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-northeast-2 - - Fn::Equals: - - Ref: AWS::Region - - ap-south-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-southeast-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-southeast-2 - - Fn::Equals: - - Ref: AWS::Region - - ca-central-1 - - Fn::Equals: - - Ref: AWS::Region - - cn-north-1 - - Fn::Equals: - - Ref: AWS::Region - - cn-northwest-1 + - Fn::Equals: [!Ref AWS::Region, af-south-1] + - Fn::Equals: [!Ref AWS::Region, ap-east-1] + - Fn::Equals: [!Ref AWS::Region, ap-northeast-1] + - Fn::Equals: [!Ref AWS::Region, ap-northeast-2] + - Fn::Equals: [!Ref AWS::Region, ap-south-1] + - Fn::Equals: [!Ref AWS::Region, ap-southeast-1] + - Fn::Equals: [!Ref AWS::Region, ap-southeast-2] + - Fn::Equals: [!Ref AWS::Region, ca-central-1] + - Fn::Equals: [!Ref AWS::Region, cn-north-1] + - Fn::Equals: [!Ref AWS::Region, cn-northwest-1] - Fn::Or: - - Fn::Equals: - - Ref: AWS::Region - - eu-central-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-north-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-south-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-west-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-west-2 - - Fn::Equals: - - Ref: AWS::Region - - eu-west-3 - - Fn::Equals: - - Ref: AWS::Region - - il-central-1 - - Fn::Equals: - - Ref: AWS::Region - - me-central-1 - - Fn::Equals: - - Ref: AWS::Region - - me-south-1 - - Fn::Equals: - - Ref: AWS::Region - - sa-east-1 + - Fn::Equals: [!Ref AWS::Region, eu-central-1] + - Fn::Equals: [!Ref AWS::Region, eu-north-1] + - Fn::Equals: [!Ref AWS::Region, eu-south-1] + - Fn::Equals: [!Ref AWS::Region, eu-west-1] + - Fn::Equals: [!Ref AWS::Region, eu-west-2] + - Fn::Equals: [!Ref AWS::Region, eu-west-3] + - Fn::Equals: [!Ref AWS::Region, il-central-1] + - Fn::Equals: [!Ref AWS::Region, me-central-1] + - Fn::Equals: [!Ref AWS::Region, me-south-1] + - Fn::Equals: [!Ref AWS::Region, sa-east-1] - Fn::Or: - - Fn::Equals: - - Ref: AWS::Region - - us-east-1 - - Fn::Equals: - - Ref: AWS::Region - - us-east-2 - - Fn::Equals: - - Ref: AWS::Region - - us-west-1 - - Fn::Equals: - - Ref: AWS::Region - - us-west-2 + - Fn::Equals: [!Ref AWS::Region, us-east-1] + - Fn::Equals: [!Ref AWS::Region, us-east-2] + - Fn::Equals: [!Ref AWS::Region, us-west-1] + - Fn::Equals: [!Ref AWS::Region, us-west-2] + Parameters: BootstrapVersion: Type: AWS::SSM::Parameter::Value Default: /cdk-bootstrap/hnb659fds/version Description: Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip] - diff --git a/javav2/example_code/entityresolution/src/test/java/EntityResTests.java b/javav2/example_code/entityresolution/src/test/java/EntityResTests.java index 03f2c75980d..c3e27fab0db 100644 --- a/javav2/example_code/entityresolution/src/test/java/EntityResTests.java +++ b/javav2/example_code/entityresolution/src/test/java/EntityResTests.java @@ -4,6 +4,7 @@ import com.example.entity.scenario.CloudFormationHelper; import com.example.entity.scenario.EntityResActions; +import java.util.concurrent.ThreadLocalRandom; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; @@ -39,7 +40,7 @@ public class EntityResTests { private static String jsonGlueTableArn = ""; - private static final String STACK_NAME = "EntityResolutionCdkStack"; + private static final String STACK_NAME = "EntityResolutionCdkStack" + ThreadLocalRandom.current().nextInt(0, 201); // 0-200 inclusive; private static final String ENTITY_RESOLUTION_ROLE_ARN_KEY = "EntityResolutionRoleArn"; private static final String GLUE_DATA_BUCKET_NAME_KEY = "GlueDataBucketName";