Issue #1060: Support GET method in HTTP connections; support x-amz-co…

…ntent-sha256 in AwsRequestSigning. Signed-off-by: Yufei Cai <yufei.cai@bosch.io>
eclipse-ditto · May 20, 2021 · b09190c · b09190c
1 parent 2fc2f4c
commit b09190c
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 32 deletions.
diff --git a/...ain/java/org/eclipse/ditto/connectivity/service/messaging/httppush/AwsRequestSigning.java b/...ain/java/org/eclipse/ditto/connectivity/service/messaging/httppush/AwsRequestSigning.java
@@ -17,6 +17,7 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -27,6 +28,7 @@
 
 import org.eclipse.ditto.base.service.UriEncoding;
 import org.eclipse.ditto.internal.utils.pubsub.ddata.Hashes;
+import org.eclipse.ditto.json.JsonParseException;
 
 import akka.NotUsed;
 import akka.actor.ActorSystem;
@@ -38,6 +40,7 @@
 import akka.http.javadsl.model.Uri;
 import akka.http.javadsl.model.headers.HttpCredentials;
 import akka.stream.javadsl.Source;
+import akka.util.ByteString;
 
 /**
  * Signing of HTTP requests to authenticate at AWS.
@@ -48,6 +51,7 @@ final class AwsRequestSigning implements RequestSigning {
     private static final char[] LOWER_CASE_HEX_CHARS = "0123456789abcdef".toCharArray();
     private static final String CONTENT_TYPE_HEADER = "content-type";
     private static final String X_AMZ_DATE_HEADER = "x-amz-date";
+    private static final String X_AMZ_CONTENT_SHA256_HEADER = "x-amz-content-sha256";
     private static final String HOST_HEADER = "host";
     private static final DateTimeFormatter DATE_STAMP_FORMATTER =
             DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneId.of("Z"));
@@ -62,18 +66,21 @@ final class AwsRequestSigning implements RequestSigning {
     private final String secretKey;
     private final boolean doubleEncodeAndNormalize;
     private final Duration timeout;
+    private final XAmzContentSha256 xAmzContentSha256;
 
     AwsRequestSigning(final ActorSystem actorSystem, final List<String> canonicalHeaderNames,
             final String region, final String service, final String accessKey, final String secretKey,
-            final boolean doubleEncodeAndNormalize, final Duration timeout) {
+            final boolean doubleEncodeAndNormalize, final Duration timeout,
+            final XAmzContentSha256 xAmzContentSha256) {
         this.actorSystem = actorSystem;
-        this.canonicalHeaderNames = toDeduplicatedSortedLowerCase(canonicalHeaderNames);
+        this.canonicalHeaderNames = toDeduplicatedSortedLowerCase(canonicalHeaderNames, xAmzContentSha256);
         this.region = region;
         this.service = service;
         this.accessKey = accessKey;
         this.secretKey = secretKey;
         this.doubleEncodeAndNormalize = doubleEncodeAndNormalize;
         this.timeout = timeout;
+        this.xAmzContentSha256 = xAmzContentSha256;
     }
 
     @Override
@@ -119,31 +126,28 @@ String getCanonicalRequest(final HttpRequest strictRequest, final Instant xAmzDa
         final String method = strictRequest.method().name();
         final String canonicalUri = getCanonicalUri(strictRequest.getUri(), doubleEncodeAndNormalize);
         final String canonicalQuery = getCanonicalQuery(strictRequest.getUri().query());
-        final String canonicalHeaders = getCanonicalHeaders(strictRequest, canonicalHeaderNames, xAmzDate);
-        final String payloadHash = sha256(strictEntity.getData().toArray());
+        final String payloadHash = getPayloadHash(strictEntity.getData());
+        final String canonicalHeaders = getCanonicalHeaders(strictRequest, xAmzDate, payloadHash);
         return String.join("\n", method, canonicalUri, canonicalQuery, canonicalHeaders, getSignedHeaders(),
                 payloadHash);
     }
 
-    private String getSignedHeaders() {
-        return String.join(";", canonicalHeaderNames);
-    }
-
-    static String sha256(final byte[] bytes) {
-        return toLowerCaseHex(Hashes.getSha256().digest(bytes));
+    String getPayloadHash(final ByteString payload) {
+        return xAmzContentSha256 == XAmzContentSha256.UNSIGNED ? "UNSIGNED-PAYLOAD" : sha256(payload.toArray());
     }
 
-    static String getCanonicalHeaders(final HttpRequest request, final Collection<String> sortedLowerCaseHeaderKeys,
-            final Instant xAmzDate) {
-        return sortedLowerCaseHeaderKeys.stream()
+    String getCanonicalHeaders(final HttpRequest request, final Instant xAmzDate, final String payloadHash) {
+        return canonicalHeaderNames.stream()
                 .map(key -> {
                     switch (key) {
                         case HOST_HEADER:
                             return HOST_HEADER + ":" + request.getUri().host().address() + "\n";
-                        case X_AMZ_DATE_HEADER:
-                            return X_AMZ_DATE_HEADER + ":" + X_AMZ_DATE_FORMATTER.format(xAmzDate) + "\n";
                         case CONTENT_TYPE_HEADER:
                             return getContentTypeAsCanonicalHeader(request);
+                        case X_AMZ_CONTENT_SHA256_HEADER:
+                            return X_AMZ_CONTENT_SHA256_HEADER + ":" + payloadHash + "\n";
+                        case X_AMZ_DATE_HEADER:
+                            return X_AMZ_DATE_HEADER + ":" + X_AMZ_DATE_FORMATTER.format(xAmzDate) + "\n";
                         default:
                             return key + streamHeaders(request, key)
                                     .map(HttpHeader::value)
@@ -154,6 +158,14 @@ static String getCanonicalHeaders(final HttpRequest request, final Collection<St
                 .collect(Collectors.joining());
     }
 
+    private String getSignedHeaders() {
+        return String.join(";", canonicalHeaderNames);
+    }
+
+    static String sha256(final byte[] bytes) {
+        return toLowerCaseHex(Hashes.getSha256().digest(bytes));
+    }
+
     static String getCanonicalQuery(final Query query) {
         return query.toMultiMap()
                 .entrySet()
@@ -212,9 +224,12 @@ static byte[] getKSigning(final byte[] kService) {
         return RequestSigning.hmacSha256(kService, "aws4_request");
     }
 
-    static Collection<String> toDeduplicatedSortedLowerCase(final List<String> strings) {
-        return strings.stream()
-                .map(String::strip)
+    static Collection<String> toDeduplicatedSortedLowerCase(final List<String> strings,
+            final XAmzContentSha256 xAmzContentSha256) {
+        final Stream<String> headers = xAmzContentSha256 == XAmzContentSha256.EXCLUDED
+                ? strings.stream()
+                : Stream.concat(strings.stream(), Stream.of(X_AMZ_CONTENT_SHA256_HEADER));
+        return headers.map(String::strip)
                 .map(String::toLowerCase)
                 .filter(s -> !s.isEmpty())
                 .sorted()
@@ -254,4 +269,15 @@ private static String toLowerCaseHex(final byte[] bytes) {
         return builder.toString();
     }
 
+    enum XAmzContentSha256 {
+        INCLUDED,
+        EXCLUDED,
+        UNSIGNED;
+
+        static XAmzContentSha256 forName(final String name) {
+            return Arrays.stream(values()).filter(option -> option.name().equals(name)).findAny()
+                    .orElseThrow(() -> new JsonParseException("The HMAC credentials parameter 'xAmzContentSha256'" +
+                            " must have one of the following as value: 'INCLUDED', 'EXCLUDED', 'UNSIGNED'."));
+        }
+    }
 }
diff --git a/...a/org/eclipse/ditto/connectivity/service/messaging/httppush/AwsRequestSigningFactory.java b/...a/org/eclipse/ditto/connectivity/service/messaging/httppush/AwsRequestSigningFactory.java
@@ -51,8 +51,11 @@ public RequestSigning create(final ActorSystem actorSystem, final HmacCredential
         final List<String> canonicalHeaders = parameters.getValue(JsonFields.CANONICAL_HEADERS)
                 .map(array -> array.stream().map(JsonValue::asString).collect(Collectors.toList()))
                 .orElse(DEFAULT_CANONICAL_HEADERS);
+        final var xAmzContentSha256 = parameters.getValue(JsonFields.X_AMZ_CONTENT_SHA256)
+                .map(AwsRequestSigning.XAmzContentSha256::forName)
+                .orElse(AwsRequestSigning.XAmzContentSha256.EXCLUDED);
         return new AwsRequestSigning(actorSystem, canonicalHeaders, region, service, accessKey, secretKey, doubleEncode,
-                TIMEOUT);
+                TIMEOUT, xAmzContentSha256);
     }
 
     /**
@@ -90,5 +93,8 @@ public static final class JsonFields {
          */
         public static JsonFieldDefinition<JsonArray> CANONICAL_HEADERS =
                 JsonFieldDefinition.ofJsonArray("canonicalHeaders");
+
+        public static JsonFieldDefinition<String> X_AMZ_CONTENT_SHA256 =
+                JsonFieldDefinition.ofString("X_AMZ_CONTENT_SHA256");
     }
 }
diff --git a/...ain/java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpPushValidator.java b/...ain/java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpPushValidator.java
@@ -14,22 +14,22 @@
 
 import static org.eclipse.ditto.internal.models.placeholders.PlaceholderFactory.newHeadersPlaceholder;
 
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
+import org.eclipse.ditto.connectivity.api.placeholders.ConnectivityPlaceholders;
 import org.eclipse.ditto.connectivity.model.Connection;
 import org.eclipse.ditto.connectivity.model.ConnectionConfigurationInvalidException;
 import org.eclipse.ditto.connectivity.model.ConnectionType;
 import org.eclipse.ditto.connectivity.model.Source;
 import org.eclipse.ditto.connectivity.model.Target;
 import org.eclipse.ditto.connectivity.service.messaging.validation.AbstractProtocolValidator;
-import org.eclipse.ditto.connectivity.api.placeholders.ConnectivityPlaceholders;
 
 import akka.actor.ActorSystem;
 import akka.http.javadsl.model.HttpMethod;
@@ -42,11 +42,11 @@ public final class HttpPushValidator extends AbstractProtocolValidator {
 
     private static final String HTTPS = "https";
     private static final String HTTP = "http";
-    private static final Collection<String> ACCEPTED_SCHEMES = Collections.unmodifiableList(Arrays.asList(HTTP, HTTPS));
+    private static final Collection<String> ACCEPTED_SCHEMES = List.of(HTTP, HTTPS);
     private static final Collection<String> SECURE_SCHEMES = Collections.singletonList(HTTPS);
 
     private static final Collection<HttpMethod> SUPPORTED_METHODS =
-            Collections.unmodifiableList(Arrays.asList(HttpMethods.PUT, HttpMethods.PATCH, HttpMethods.POST));
+            List.of(HttpMethods.PUT, HttpMethods.PATCH, HttpMethods.POST, HttpMethods.GET);
 
     private static final String SUPPORTED_METHOD_NAMES = SUPPORTED_METHODS.stream()
             .map(HttpMethod::name)

diff --git a/...java/org/eclipse/ditto/connectivity/service/messaging/httppush/AwsRequestSigningTest.java b/...java/org/eclipse/ditto/connectivity/service/messaging/httppush/AwsRequestSigningTest.java
@@ -16,9 +16,9 @@
 
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 import org.bouncycastle.util.encoders.Hex;
 import org.junit.After;
@@ -35,6 +35,7 @@
 import akka.http.javadsl.model.headers.HttpCredentials;
 import akka.stream.javadsl.Sink;
 import akka.testkit.javadsl.TestKit;
+import akka.util.ByteString;
 
 /**
  * Test cases for AWS request signing.
@@ -53,6 +54,9 @@ public final class AwsRequestSigningTest {
 
     private static final String BODY = "The quick brown fox jumped over the lazy dog";
 
+    private static final AwsRequestSigning.XAmzContentSha256 X_AMZ_CONTENT_SHA256 =
+            AwsRequestSigning.XAmzContentSha256.EXCLUDED;
+
     private final ActorSystem actorSystem = ActorSystem.create();
 
     @After
@@ -106,8 +110,8 @@ public void testCanonicalHeaders() {
                 // can't use .addHeader because it _prepends_ the header to the list
                 .withHeaders(headesInSequence);
 
-        final Collection<String> canonicalHeaderNames = AwsRequestSigning.toDeduplicatedSortedLowerCase(
-                List.of("Host", "My-header1", "X-Amz-Date", "my-headER2", "CONTENT-TYPE", "nonexistent-header"));
+        final List<String> signedHeaders =
+                List.of("Host", "My-header1", "X-Amz-Date", "my-headER2", "CONTENT-TYPE", "nonexistent-header");
 
         final String expectedCanonicalHeaders =
                 "content-type:application/x-www-form-urlencoded; charset=UTF-8\n" +
@@ -117,13 +121,41 @@ public void testCanonicalHeaders() {
                         "nonexistent-header:\n" +
                         "x-amz-date:20150830T123600Z\n";
 
+        final AwsRequestSigning underTest =
+                new AwsRequestSigning(actorSystem, signedHeaders, REGION_NAME, SERVICE_NAME, ACCESS_KEY,
+                        SECRET_KEY, true, Duration.ofSeconds(10), X_AMZ_CONTENT_SHA256);
         final String canonicalHeaders =
-                AwsRequestSigning.getCanonicalHeaders(request, canonicalHeaderNames,
-                        Instant.parse("2015-08-30T12:36:00Z"));
+                underTest.getCanonicalHeaders(request, Instant.parse("2015-08-30T12:36:00Z"), "UNSIGNED-PAYLOAD");
 
         assertThat(canonicalHeaders).isEqualTo(expectedCanonicalHeaders);
     }
 
+    @Test
+    public void testXAmzContentSha256() {
+        final HttpRequest request = getSampleHttpRequest();
+
+        final Function<AwsRequestSigning.XAmzContentSha256, AwsRequestSigning> creator = xAmzContentSha256 ->
+                new AwsRequestSigning(actorSystem, List.of("host"), REGION_NAME, SERVICE_NAME, ACCESS_KEY,
+                        SECRET_KEY, true, Duration.ofSeconds(10), xAmzContentSha256);
+
+        final var included = creator.apply(AwsRequestSigning.XAmzContentSha256.INCLUDED);
+        final var excluded = creator.apply(AwsRequestSigning.XAmzContentSha256.EXCLUDED);
+        final var unsigned = creator.apply(AwsRequestSigning.XAmzContentSha256.UNSIGNED);
+
+        assertThat(included.getCanonicalHeaders(request, Instant.parse("2015-08-30T12:36:00Z"),
+                included.getPayloadHash(ByteString.emptyByteString())))
+                .isEqualTo("host:www.example.com\n" +
+                        "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n");
+
+        assertThat(excluded.getCanonicalHeaders(request, Instant.parse("2015-08-30T12:36:00Z"),
+                excluded.getPayloadHash(ByteString.emptyByteString())))
+                .isEqualTo("host:www.example.com\n");
+
+        assertThat(unsigned.getCanonicalHeaders(request, Instant.parse("2015-08-30T12:36:00Z"),
+                unsigned.getPayloadHash(ByteString.emptyByteString())))
+                .isEqualTo("host:www.example.com\nx-amz-content-sha256:UNSIGNED-PAYLOAD\n");
+    }
+
     @Test
     public void testRequestSignature() {
         final HttpRequest requestToSign = getSampleHttpRequest();
@@ -202,7 +234,7 @@ private static HttpRequest getSampleHttpRequest() {
 
     private HttpRequest signRequest(final HttpRequest originalRequest) {
         return new AwsRequestSigning(actorSystem, List.of("host", "x-amz-date"), REGION_NAME, SERVICE_NAME, ACCESS_KEY,
-                SECRET_KEY, true, Duration.ofSeconds(10))
+                SECRET_KEY, true, Duration.ofSeconds(10), X_AMZ_CONTENT_SHA256)
                 .sign(originalRequest, X_AMZ_DATE)
                 .runWith(Sink.head(), actorSystem)
                 .toCompletableFuture()
@@ -211,13 +243,13 @@ private HttpRequest signRequest(final HttpRequest originalRequest) {
 
     private String getStringToSign(final HttpRequest httpRequest) {
         return new AwsRequestSigning(actorSystem, List.of("host", "x-amz-date"), REGION_NAME, SERVICE_NAME, ACCESS_KEY,
-                SECRET_KEY, true, Duration.ofSeconds(10))
+                SECRET_KEY, true, Duration.ofSeconds(10), X_AMZ_CONTENT_SHA256)
                 .getStringToSign(httpRequest, X_AMZ_DATE, true);
     }
 
     private String getCanonicalRequest(final HttpRequest httpRequest) {
         return new AwsRequestSigning(actorSystem, List.of("host", "x-amz-date"), REGION_NAME, SERVICE_NAME, ACCESS_KEY,
-                SECRET_KEY, true, Duration.ofSeconds(10))
+                SECRET_KEY, true, Duration.ofSeconds(10), X_AMZ_CONTENT_SHA256)
                 .getCanonicalRequest(httpRequest, X_AMZ_DATE, true);
     }
 

diff --git a/...java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpPushValidatorTest.java b/...java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpPushValidatorTest.java
@@ -97,13 +97,13 @@ public void testValidTargetAddress() {
         underTest.validate(getConnectionWithTarget("PUT:events#{{topic:full}}"), emptyDittoHeaders, actorSystem);
         underTest.validate(getConnectionWithTarget("POST:ditto?{{header:x}}"), emptyDittoHeaders, actorSystem);
         underTest.validate(getConnectionWithTarget("POST:"), emptyDittoHeaders, actorSystem);
+        underTest.validate(getConnectionWithTarget("GET:foo"), emptyDittoHeaders, actorSystem);
     }
 
     @Test
     public void testInvalidTargetAddress() {
         verifyConnectionConfigurationInvalidExceptionIsThrown(getConnectionWithTarget(""));
         verifyConnectionConfigurationInvalidExceptionIsThrown(getConnectionWithTarget("events"));
-        verifyConnectionConfigurationInvalidExceptionIsThrown(getConnectionWithTarget("GET:foo"));
         verifyConnectionConfigurationInvalidExceptionIsThrown(getConnectionWithTarget("DELETE:/bar"));
     }