From 6b03b1e52fec0aa4cb6af464318f4312aee1be44 Mon Sep 17 00:00:00 2001 From: Anton Mokhovikov Date: Tue, 5 May 2020 00:01:29 -0700 Subject: [PATCH 1/4] added compression to handler response stream --- pom.xml | 6 +++ .../amazon/cloudformation/LambdaWrapper.java | 3 +- .../cloudformation/resource/Serializer.java | 40 ++++++++++++++++++- .../scheduler/CloudWatchScheduler.java | 7 ++-- .../scheduler/CloudWatchSchedulerTest.java | 11 ++--- 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 2e03ca96..2f1b9f85 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,12 @@ + + + commons-codec + commons-codec + 1.14 + software.amazon.cloudformation diff --git a/src/main/java/software/amazon/cloudformation/LambdaWrapper.java b/src/main/java/software/amazon/cloudformation/LambdaWrapper.java index 653c13da..f992100f 100644 --- a/src/main/java/software/amazon/cloudformation/LambdaWrapper.java +++ b/src/main/java/software/amazon/cloudformation/LambdaWrapper.java @@ -249,7 +249,8 @@ public void handleRequest(final InputStream inputStream, final OutputStream outp throw new TerminalException("No request object received"); } - String input = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + String input = this.serializer.decompress(IOUtils.toString(inputStream, StandardCharsets.UTF_8)); + JSONObject rawInput = new JSONObject(new JSONTokener(input)); // deserialize incoming payload to modelled request diff --git a/src/main/java/software/amazon/cloudformation/resource/Serializer.java b/src/main/java/software/amazon/cloudformation/resource/Serializer.java index a53f903f..2c4bdc1d 100644 --- a/src/main/java/software/amazon/cloudformation/resource/Serializer.java +++ b/src/main/java/software/amazon/cloudformation/resource/Serializer.java @@ -14,6 +14,7 @@ */ package software.amazon.cloudformation.resource; +import com.amazonaws.util.IOUtils; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -22,14 +23,24 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import org.apache.commons.codec.binary.Base64; import software.amazon.cloudformation.proxy.aws.AWSServiceSerdeModule; public class Serializer { + public static final String COMPRESSED = "compressed"; private static final ObjectMapper OBJECT_MAPPER; - private static final ObjectMapper STRICT_OBJECT_MAPPER; + private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() { + }; /** * Configures the specified ObjectMapper with the (de)serialization behaviours @@ -76,10 +87,37 @@ public String serialize(final T modelObject) throws JsonProcessingException return OBJECT_MAPPER.writeValueAsString(modelObject); } + public String compress(final String modelInput) throws IOException { + final Map map = new HashMap<>(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(byteArrayOutputStream); + gzip.write(modelInput.getBytes(StandardCharsets.UTF_8)); + gzip.close(); + map.put(COMPRESSED, Base64.encodeBase64String(byteArrayOutputStream.toByteArray())); + return OBJECT_MAPPER.writeValueAsString(map); + } + public T deserialize(final String s, final TypeReference reference) throws IOException { return OBJECT_MAPPER.readValue(s, reference); } + public String decompress(final String s) { + try { + final Map map = deserialize(s, MAP_TYPE_REFERENCE); + + if (!map.containsKey(COMPRESSED)) { + return s; + } + + final byte[] bytes = Base64.decodeBase64(map.get(COMPRESSED)); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); + return new String(IOUtils.toByteArray(gzipInputStream), StandardCharsets.UTF_8); + } catch (IOException e) { + return s; + } + } + public T deserializeStrict(final String s, final TypeReference reference) throws IOException { return STRICT_OBJECT_MAPPER.readValue(s, reference); } diff --git a/src/main/java/software/amazon/cloudformation/scheduler/CloudWatchScheduler.java b/src/main/java/software/amazon/cloudformation/scheduler/CloudWatchScheduler.java index b0dc491b..55b18263 100644 --- a/src/main/java/software/amazon/cloudformation/scheduler/CloudWatchScheduler.java +++ b/src/main/java/software/amazon/cloudformation/scheduler/CloudWatchScheduler.java @@ -14,7 +14,7 @@ */ package software.amazon.cloudformation.scheduler; -import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.IOException; import java.util.Objects; import java.util.UUID; import lombok.Data; @@ -101,14 +101,15 @@ public void rescheduleAfterMinutes(final String functionA String jsonRequest; try { // expect return type to be non-null - jsonRequest = serializer.serialize(handlerRequest); - } catch (JsonProcessingException e) { + jsonRequest = serializer.compress(serializer.serialize(handlerRequest)); + } catch (IOException e) { throw new TerminalException("Unable to serialize the request for callback", e); } this.log(String.format("Scheduling re-invoke at %s (%s)%n", cronRule, rescheduleId)); PutRuleRequest putRuleRequest = PutRuleRequest.builder().name(ruleName).scheduleExpression(cronRule) .state(RuleState.ENABLED).build(); + this.client.putRule(putRuleRequest); Target target = Target.builder().arn(functionArn).id(targetId).input(jsonRequest).build(); diff --git a/src/test/java/software/amazon/cloudformation/scheduler/CloudWatchSchedulerTest.java b/src/test/java/software/amazon/cloudformation/scheduler/CloudWatchSchedulerTest.java index 1e7a8b9e..7ace62ea 100644 --- a/src/test/java/software/amazon/cloudformation/scheduler/CloudWatchSchedulerTest.java +++ b/src/test/java/software/amazon/cloudformation/scheduler/CloudWatchSchedulerTest.java @@ -21,9 +21,9 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.util.Arrays; +import java.io.IOException; +import java.util.Collections; import java.util.List; -import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -148,7 +148,7 @@ public void test_cleanupCloudWatchEventsWithErrorDeletingRule() { } @Test - public void test_rescheduleAfterMinutes_1MinuteFloor() { + public void test_rescheduleAfterMinutes_1MinuteFloor() throws IOException { final CloudWatchEventsProvider provider = mock(CloudWatchEventsProvider.class); final CloudWatchEventsClient client = getCloudWatchEvents(); when(provider.get()).thenReturn(client); @@ -166,8 +166,9 @@ public void test_rescheduleAfterMinutes_1MinuteFloor() { verify(requestContext, times(1)).setCloudWatchEventsRuleName(startsWith("reinvoke-handler-")); verify(requestContext, times(1)).setCloudWatchEventsTargetId(startsWith("reinvoke-target-")); - final List targetMatchers = Arrays - .asList(new TargetMatcher(FUNCTION_ARN, "reinvoke-target-", new JSONObject(request).toString())); + final List targetMatchers = Collections.singletonList( + new TargetMatcher(FUNCTION_ARN, "reinvoke-target-", serializer.compress(serializer.serialize(request)))); + verify(client, times(1)) .putTargets(argThat(new PutTargetsRequestMatcher("reinvoke-handler-", new TargetsListMatcher(targetMatchers)))); verify(client, times(1)) From 49e36fb94d35b36d6323cfaa468598e27bc87ce1 Mon Sep 17 00:00:00 2001 From: Anton Mokhovikov Date: Wed, 6 May 2020 13:53:26 -0700 Subject: [PATCH 2/4] added try-block reference --- .../cloudformation/resource/Serializer.java | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/main/java/software/amazon/cloudformation/resource/Serializer.java b/src/main/java/software/amazon/cloudformation/resource/Serializer.java index 2c4bdc1d..d8838010 100644 --- a/src/main/java/software/amazon/cloudformation/resource/Serializer.java +++ b/src/main/java/software/amazon/cloudformation/resource/Serializer.java @@ -36,10 +36,11 @@ public class Serializer { - public static final String COMPRESSED = "compressed"; + public static final String COMPRESSED = "__COMPRESSED__"; + private static final String COMPRESSION_METHOD = "__COMPRESSION_METHOD__"; private static final ObjectMapper OBJECT_MAPPER; private static final ObjectMapper STRICT_OBJECT_MAPPER; - private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() { + private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() { }; /** @@ -87,13 +88,27 @@ public String serialize(final T modelObject) throws JsonProcessingException return OBJECT_MAPPER.writeValueAsString(modelObject); } + /* + * public String compress(final String modelInput) throws IOException { + * final Map map = new HashMap<>(); ByteArrayOutputStream + * byteArrayOutputStream = new ByteArrayOutputStream(); GZIPOutputStream gzip = + * new GZIPOutputStream(byteArrayOutputStream); + * gzip.write(modelInput.getBytes(StandardCharsets.UTF_8)); gzip.close(); + * map.put(COMPRESSED, + * Base64.encodeBase64String(byteArrayOutputStream.toByteArray())); + * map.put(COMPRESSION_METHOD, "gzip_base64"); return + * OBJECT_MAPPER.writeValueAsString(map); } + */ + public String compress(final String modelInput) throws IOException { final Map map = new HashMap<>(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - GZIPOutputStream gzip = new GZIPOutputStream(byteArrayOutputStream); - gzip.write(modelInput.getBytes(StandardCharsets.UTF_8)); - gzip.close(); - map.put(COMPRESSED, Base64.encodeBase64String(byteArrayOutputStream.toByteArray())); + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + try (GZIPOutputStream gzip = new GZIPOutputStream(byteArrayOutputStream)) { + gzip.write(modelInput.getBytes(StandardCharsets.UTF_8)); + } + map.put(COMPRESSED, Base64.encodeBase64String(byteArrayOutputStream.toByteArray())); + map.put(COMPRESSION_METHOD, "gzip_base64"); + } return OBJECT_MAPPER.writeValueAsString(map); } @@ -101,20 +116,17 @@ public T deserialize(final String s, final TypeReference reference) throw return OBJECT_MAPPER.readValue(s, reference); } - public String decompress(final String s) { - try { - final Map map = deserialize(s, MAP_TYPE_REFERENCE); + public String decompress(final String s) throws IOException { + final Map map = deserialize(s, MAP_TYPE_REFERENCE); - if (!map.containsKey(COMPRESSED)) { - return s; - } + if (!map.containsKey(COMPRESSED)) { + return s; + } - final byte[] bytes = Base64.decodeBase64(map.get(COMPRESSED)); - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); - GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); + final byte[] bytes = Base64.decodeBase64((String) map.get(COMPRESSED)); + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream);) { return new String(IOUtils.toByteArray(gzipInputStream), StandardCharsets.UTF_8); - } catch (IOException e) { - return s; } } From 637343d52fcb01dd5b3436d2aa64251710b3cdb7 Mon Sep 17 00:00:00 2001 From: Anton Mokhovikov Date: Thu, 7 May 2020 00:28:58 -0700 Subject: [PATCH 3/4] added compression field as static final --- .../cloudformation/resource/Serializer.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/main/java/software/amazon/cloudformation/resource/Serializer.java b/src/main/java/software/amazon/cloudformation/resource/Serializer.java index d8838010..435a63b2 100644 --- a/src/main/java/software/amazon/cloudformation/resource/Serializer.java +++ b/src/main/java/software/amazon/cloudformation/resource/Serializer.java @@ -38,6 +38,7 @@ public class Serializer { public static final String COMPRESSED = "__COMPRESSED__"; private static final String COMPRESSION_METHOD = "__COMPRESSION_METHOD__"; + private static final String COMPRESSION = "gzip_base64"; private static final ObjectMapper OBJECT_MAPPER; private static final ObjectMapper STRICT_OBJECT_MAPPER; private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() { @@ -88,18 +89,6 @@ public String serialize(final T modelObject) throws JsonProcessingException return OBJECT_MAPPER.writeValueAsString(modelObject); } - /* - * public String compress(final String modelInput) throws IOException { - * final Map map = new HashMap<>(); ByteArrayOutputStream - * byteArrayOutputStream = new ByteArrayOutputStream(); GZIPOutputStream gzip = - * new GZIPOutputStream(byteArrayOutputStream); - * gzip.write(modelInput.getBytes(StandardCharsets.UTF_8)); gzip.close(); - * map.put(COMPRESSED, - * Base64.encodeBase64String(byteArrayOutputStream.toByteArray())); - * map.put(COMPRESSION_METHOD, "gzip_base64"); return - * OBJECT_MAPPER.writeValueAsString(map); } - */ - public String compress(final String modelInput) throws IOException { final Map map = new HashMap<>(); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { @@ -107,7 +96,7 @@ public String compress(final String modelInput) throws IOException { gzip.write(modelInput.getBytes(StandardCharsets.UTF_8)); } map.put(COMPRESSED, Base64.encodeBase64String(byteArrayOutputStream.toByteArray())); - map.put(COMPRESSION_METHOD, "gzip_base64"); + map.put(COMPRESSION_METHOD, COMPRESSION); } return OBJECT_MAPPER.writeValueAsString(map); } From 62c77d02b43d96629944d61a4d921fcbe7646fd7 Mon Sep 17 00:00:00 2001 From: Anton Mokhovikov Date: Thu, 7 May 2020 00:33:17 -0700 Subject: [PATCH 4/4] modified var naming --- .../software/amazon/cloudformation/resource/Serializer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/software/amazon/cloudformation/resource/Serializer.java b/src/main/java/software/amazon/cloudformation/resource/Serializer.java index 435a63b2..6fda8a07 100644 --- a/src/main/java/software/amazon/cloudformation/resource/Serializer.java +++ b/src/main/java/software/amazon/cloudformation/resource/Serializer.java @@ -38,7 +38,7 @@ public class Serializer { public static final String COMPRESSED = "__COMPRESSED__"; private static final String COMPRESSION_METHOD = "__COMPRESSION_METHOD__"; - private static final String COMPRESSION = "gzip_base64"; + private static final String COMPRESSION_GZIP_BASE64 = "gzip_base64"; private static final ObjectMapper OBJECT_MAPPER; private static final ObjectMapper STRICT_OBJECT_MAPPER; private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() { @@ -96,7 +96,7 @@ public String compress(final String modelInput) throws IOException { gzip.write(modelInput.getBytes(StandardCharsets.UTF_8)); } map.put(COMPRESSED, Base64.encodeBase64String(byteArrayOutputStream.toByteArray())); - map.put(COMPRESSION_METHOD, COMPRESSION); + map.put(COMPRESSION_METHOD, COMPRESSION_GZIP_BASE64); } return OBJECT_MAPPER.writeValueAsString(map); }