-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
馃毃 Add SSL documentation and check logic for S3 Destination 馃毃 (#17340)
* Adds logic to fail upon non-deterministic custom S3 endpoint and documentation for insecure settings * Reused config factory settings to a single static variable * Updated error message and example in the spec.json to match expectation of secured endpoint * Added validation check within the base s3 * Integrated AdaptiveDestinationRunner with S3Destination * Reduced visibility for testing and fixed AdaptiveDestinationRunner issue * Adds speicifc secure protocol with S3 and empty endpoint check * Bumps docker version and adds comments and clearer string methods * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
- Loading branch information
1 parent
bb6dff5
commit 1d956df
Showing
15 changed files
with
175 additions
and
37 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 17 additions & 0 deletions
17
...tination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationRunner.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.s3; | ||
|
|
||
| import io.airbyte.integrations.base.adaptive.AdaptiveDestinationRunner; | ||
|
|
||
| public class S3DestinationRunner { | ||
|
|
||
| public static void main(final String[] args) throws Exception { | ||
| AdaptiveDestinationRunner.baseOnEnv() | ||
| .withOssDestination(S3Destination::new) | ||
| .withCloudDestination(S3DestinationStrictEncrypt::new) | ||
| .run(args); | ||
| } | ||
| } |
35 changes: 35 additions & 0 deletions
35
...n-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.s3; | ||
|
|
||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.google.common.annotations.VisibleForTesting; | ||
| import io.airbyte.protocol.models.AirbyteConnectionStatus; | ||
| import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; | ||
|
|
||
| public class S3DestinationStrictEncrypt extends S3Destination { | ||
|
|
||
| public S3DestinationStrictEncrypt() { | ||
| super(); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| protected S3DestinationStrictEncrypt(final S3DestinationConfigFactory configFactory) { | ||
| super(configFactory); | ||
| } | ||
|
|
||
| @Override | ||
| public AirbyteConnectionStatus check(final JsonNode config) { | ||
| final S3DestinationConfig destinationConfig = this.configFactory.getS3DestinationConfig(config, super.storageProvider()); | ||
|
|
||
| // Fails early to avoid extraneous validations checks if custom endpoint is not secure | ||
| if (!S3BaseChecks.testCustomEndpointSecured(destinationConfig.getEndpoint())) { | ||
| return new AirbyteConnectionStatus() | ||
| .withStatus(Status.FAILED) | ||
| .withMessage("Custom endpoint does not use HTTPS"); | ||
| } | ||
| return super.check(config); | ||
| } | ||
| } |
76 changes: 76 additions & 0 deletions
76
.../src/test/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncryptTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.s3; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
| import static org.mockito.ArgumentMatchers.any; | ||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.when; | ||
|
|
||
| import com.amazonaws.services.s3.AmazonS3; | ||
| import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; | ||
| import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; | ||
| import com.amazonaws.services.s3.model.UploadPartRequest; | ||
| import com.amazonaws.services.s3.model.UploadPartResult; | ||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import io.airbyte.protocol.models.AirbyteConnectionStatus; | ||
| import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; | ||
| import org.junit.jupiter.api.BeforeEach; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| public class S3DestinationStrictEncryptTest { | ||
|
|
||
| private AmazonS3 s3; | ||
| private S3DestinationConfigFactory factoryConfig; | ||
|
|
||
| @BeforeEach | ||
| public void setup() { | ||
| s3 = mock(AmazonS3.class); | ||
| final InitiateMultipartUploadResult uploadResult = mock(InitiateMultipartUploadResult.class); | ||
| final UploadPartResult uploadPartResult = mock(UploadPartResult.class); | ||
| when(s3.uploadPart(any(UploadPartRequest.class))).thenReturn(uploadPartResult); | ||
| when(s3.initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(uploadResult); | ||
|
|
||
| factoryConfig = new S3DestinationConfigFactory() { | ||
| public S3DestinationConfig getS3DestinationConfig(final JsonNode config, final StorageProvider storageProvider) { | ||
| return S3DestinationConfig.create("fake-bucket", "fake-bucketPath", "fake-region") | ||
| .withEndpoint("https://s3.example.com") | ||
| .withAccessKeyCredential("fake-accessKeyId", "fake-secretAccessKey") | ||
| .withS3Client(s3) | ||
| .get(); | ||
| } | ||
| }; | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * Test that checks if user is using a connection that is HTTPS only | ||
| */ | ||
| @Test | ||
| public void checksCustomEndpointIsHttpsOnly() { | ||
| final S3Destination destinationWithHttpsOnlyEndpoint = new S3DestinationStrictEncrypt(factoryConfig); | ||
| final AirbyteConnectionStatus status = destinationWithHttpsOnlyEndpoint.check(null); | ||
| assertEquals(Status.SUCCEEDED, status.getStatus(), "custom endpoint did not contain `s3-accesspoint`"); | ||
| } | ||
|
|
||
| /** | ||
| * Test that checks if user is using a connection that is deemed insecure since it does not always enforce HTTPS only | ||
| * <p>https://docs.aws.amazon.com/general/latest/gr/s3.html</p> | ||
| */ | ||
| @Test | ||
| public void checksCustomEndpointIsNotHttpsOnly() { | ||
| final S3Destination destinationWithStandardUnsecuredEndpoint = new S3DestinationStrictEncrypt(new S3DestinationConfigFactory() { | ||
| public S3DestinationConfig getS3DestinationConfig(final JsonNode config, final StorageProvider storageProvider) { | ||
| return S3DestinationConfig.create("fake-bucket", "fake-bucketPath", "fake-region") | ||
| .withEndpoint("s3.us-west-1.amazonaws.com") | ||
| .withAccessKeyCredential("fake-accessKeyId", "fake-secretAccessKey") | ||
| .withS3Client(s3) | ||
| .get(); | ||
| } | ||
| }); | ||
| final AirbyteConnectionStatus status = destinationWithStandardUnsecuredEndpoint.check(null); | ||
| assertEquals(Status.FAILED, status.getStatus()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters