From 096733f5da8848f701f54509db2e8c2d68c2c9be Mon Sep 17 00:00:00 2001 From: apatterson Date: Wed, 17 Apr 2019 10:16:14 -0700 Subject: [PATCH] IDTS for AWS S3 v1.0.0 --- README.md | 12 +- awss3examples/.gitignore | 3 - awss3examples/pom.xml | 67 --- .../ionicsecurity/examples/S3SampleApp.java | 529 ----------------- .../examples/S3TransferManagerSample.java | 136 ----- examples/pom.xml | 69 +++ {awss3examples => examples}/run.bat | 0 {awss3examples => examples}/run.sh | 0 .../cloudstorage/samples/S3SampleApp.java | 540 ++++++++++++++++++ .../samples/S3TransferManagerSample.java | 137 +++++ pom.xml | 259 +++++---- src/main/java-templates/Version.java | 23 +- .../cloudstorage/awss3/IonicAgentFactory.java | 86 +++ .../IonicEncryptionMaterialsProvider.java | 177 +++--- .../awss3/IonicS3EncryptionClient.java | 224 +++++--- .../awss3/IonicS3EncryptionClientBuilder.java | 59 +- .../awss3/IonicS3EncryptionClientParams.java | 35 +- .../ionicsecurity/ipcs/awss3/ISAgentPool.java | 123 ---- ...yptionMaterialsProviderFunctionalTest.java | 74 +++ ...icEncryptionMaterialsProviderUnitTest.java | 84 +++ ...onicS3EncryptionClientBuilderUnitTest.java | 94 +++ .../IonicS3EncryptionClientDeniedTest.java | 75 +++ ...IonicS3EncryptionClientFunctionalTest.java | 357 ++++++++++++ .../cloudstorage/awss3/SkipFailListener.java | 23 + .../ionic/cloudstorage/awss3/TestUtils.java | 196 +++++++ .../awss3/TransferManagerFunctionalTest.java | 182 ++++++ .../IonicEncryptionMaterialsProviderTest.java | 117 ---- .../IonicS3EncryptionClientBuilderTest.java | 70 --- .../awss3/IonicS3EncryptionClientTest.java | 368 ------------ .../ipcs/awss3/TransferManagerTest.java | 111 ---- src/test/resources/largeFile.txt | 0 src/test/resources/metaDataDest.txt | 0 src/test/resources/testFile.txt | 1 - src/test/resources/transferTarget1.txt | 0 34 files changed, 2327 insertions(+), 1904 deletions(-) delete mode 100644 awss3examples/.gitignore delete mode 100644 awss3examples/pom.xml delete mode 100644 awss3examples/src/main/java/com/ionicsecurity/examples/S3SampleApp.java delete mode 100644 awss3examples/src/main/java/com/ionicsecurity/examples/S3TransferManagerSample.java create mode 100644 examples/pom.xml rename {awss3examples => examples}/run.bat (100%) rename {awss3examples => examples}/run.sh (100%) create mode 100644 examples/src/main/java/com/ionic/cloudstorage/samples/S3SampleApp.java create mode 100644 examples/src/main/java/com/ionic/cloudstorage/samples/S3TransferManagerSample.java create mode 100644 src/main/java/com/ionic/cloudstorage/awss3/IonicAgentFactory.java rename src/main/java/com/{ionicsecurity/ipcs => ionic/cloudstorage}/awss3/IonicEncryptionMaterialsProvider.java (62%) rename src/main/java/com/{ionicsecurity/ipcs => ionic/cloudstorage}/awss3/IonicS3EncryptionClient.java (63%) rename src/main/java/com/{ionicsecurity/ipcs => ionic/cloudstorage}/awss3/IonicS3EncryptionClientBuilder.java (64%) rename src/main/java/com/{ionicsecurity/ipcs => ionic/cloudstorage}/awss3/IonicS3EncryptionClientParams.java (58%) delete mode 100644 src/main/java/com/ionicsecurity/ipcs/awss3/ISAgentPool.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderFunctionalTest.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderUnitTest.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientBuilderUnitTest.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientDeniedTest.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientFunctionalTest.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/SkipFailListener.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/TestUtils.java create mode 100644 src/test/java/com/ionic/cloudstorage/awss3/TransferManagerFunctionalTest.java delete mode 100644 src/test/java/com/ionicsecurity/ipcs/awss3/IonicEncryptionMaterialsProviderTest.java delete mode 100644 src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientBuilderTest.java delete mode 100644 src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientTest.java delete mode 100644 src/test/java/com/ionicsecurity/ipcs/awss3/TransferManagerTest.java delete mode 100644 src/test/resources/largeFile.txt delete mode 100644 src/test/resources/metaDataDest.txt delete mode 100644 src/test/resources/testFile.txt delete mode 100644 src/test/resources/transferTarget1.txt diff --git a/README.md b/README.md index a56b473..01575b3 100644 --- a/README.md +++ b/README.md @@ -64,13 +64,13 @@ Replace `your_access_key_id` and `your_secret_access_key` with the keys created ### Build the Project To build and install the Ionic S3 sdk `mvn install` -To build the IonicS3SampleApp from /awss3examples `mvn package` +To build the IonicS3SampleApp from ./examples `mvn package` ### Usage #### Sample App -After the build, a fat JAR of the sample app is produced at `awss3examples/target/IonicS3SampleApp.jar`. +After the build, a fat JAR of the sample app is produced at `./examples/target/IonicS3SampleApp.jar`. Ensure that your Ionic device credentials can be located at `${user.home}/.ionicsecurity/profiles.pt`. @@ -78,8 +78,8 @@ The user can run the program from /awss3examples with either "put" or "get" comm * `./run.sh putString []` * `./run.sh putFile []` * `./run.sh putMultipart []` -* `./run.sh getFile [-m] ` -* `./run.sh getString [-m] ` +* `./run.sh getFile ` +* `./run.sh getString ` Windows users should use ./run.bat instead Note: S3SampleApp does not protect against invalid entry of AWS S3 bucket names or Object Keys @@ -90,6 +90,6 @@ Note: S3SampleApp does not protect against invalid entry of AWS S3 bucket names #### Using the Library -Using the library as a developer is documented in the `docs/content/` directories. -The JAR for use here is produced in `ionics3/target/`, such as `ionics3/target/ionics3-0.7.2.jar`. +Developer documentation for this library can be found at https://dev.ionic.com/integrations/aws-s3. +The JAR for use here is produced in `./target/`, such as `./target/ionic-for-aws-java-sdk-s3-1.0.0.jar`. This JAR is thin, and needs the Ionic SDK and the AWS SDKs available during builds that use it. diff --git a/awss3examples/.gitignore b/awss3examples/.gitignore deleted file mode 100644 index e2b668c..0000000 --- a/awss3examples/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/target/ -/bin/ -dependency-reduced-pom.xml diff --git a/awss3examples/pom.xml b/awss3examples/pom.xml deleted file mode 100644 index 8595dbe..0000000 --- a/awss3examples/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - 4.0.0 - com.ionicsecurity.ipcs.awss3 - awss3examples - 0.9.0 - jar - https://ionic.com - - UTF-8 - - - - - org.apache.maven.plugins - maven-compiler-plugin - 2.5.1 - - 1.8 - 1.8 - true - true - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.0 - - - package - - shade - - - IonicS3SampleApp - - - com.ionicsecurity.examples.S3SampleApp - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - - - com.ionicsecurity.ipcs.awss3 - ionics3 - 0.9.0 - - - diff --git a/awss3examples/src/main/java/com/ionicsecurity/examples/S3SampleApp.java b/awss3examples/src/main/java/com/ionicsecurity/examples/S3SampleApp.java deleted file mode 100644 index 2ef6743..0000000 --- a/awss3examples/src/main/java/com/ionicsecurity/examples/S3SampleApp.java +++ /dev/null @@ -1,529 +0,0 @@ -/* - * S3SampleApp.java The purpose of this project is to store an object in AWS S3 with client-side - * Ionic protection. This code is an example of what clients would use programmatically to - * incorporate the Ionic platform into their S3 use cases. - * - * (c) 2017-2018 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as - * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy - * (https://www.ionic.com/privacy-notice/). Derived in part from AWS Sample S3 Project, - * S3Sample.java. - */ - -package com.ionicsecurity.examples; - -import java.io.IOException; -import java.nio.file.InvalidPathException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.util.Map; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import com.ionic.sdk.agent.data.MetadataMap; -import com.ionic.sdk.agent.key.KeyAttributesMap; -import com.ionic.sdk.agent.request.createkey.CreateKeysRequest; -import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; -import com.ionic.sdk.error.IonicException; -import com.ionicsecurity.ipcs.awss3.IonicEncryptionMaterialsProvider; -import com.ionicsecurity.ipcs.awss3.IonicS3EncryptionClient; -import com.ionicsecurity.ipcs.awss3.IonicS3EncryptionClientBuilder; -import com.ionicsecurity.ipcs.awss3.Version; -import com.amazonaws.SdkClientException; -import com.amazonaws.services.s3.AmazonS3Encryption; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.util.IOUtils; -import com.amazonaws.util.StringUtils; - -public class S3SampleApp { - - enum Action { - GETSTRING("getString"), - GETFILE("getFile"), - PUTSTRING("putString"), - PUTFILE("putFile"), - PUTMULTIPART("putMultipart"), - VERSION("version"),; - - final String str; - - Action(String name) { - this.str = name; - } - } - - public static final int MAX_AWS_SDK_CHUNKS = 10000; // AWS S3 limits multipart uploads to 10000 parts - public static final boolean useSandbox = true; // if true limit file paths to within user's home dir - private static final String HOME = System.getProperty("user.home"); - - - static void putString(String bucketName, String objectKey, String objectContent, - IonicS3EncryptionClient s3, KeyAttributesMap attributes) { - System.out.println("Putting object as string in specified S3 bucket"); - if (attributes != null) { - s3.putObject(bucketName, objectKey, objectContent, new CreateKeysRequest.Key("", 1, attributes)); - } else { - s3.putObject(bucketName, objectKey, objectContent); - } - } - - static void putFile(String bucketName, String objectKey, String filePath, IonicS3EncryptionClient s3, - KeyAttributesMap attributes) { - - String srcFilePathStr = getCanonicalPathString(filePath); - - if ((srcFilePathStr == null) || (srcFilePathStr.isEmpty())) { - System.err.println("No filepath specified"); - return; - } - - // Sandbox within user home - if ((useSandbox) && (!srcFilePathStr.startsWith(HOME))) { - System.err.println("Filepath outside of user home"); - return; - } - - Path srcFilePath = Paths.get(srcFilePathStr); - - if (!Files.exists(srcFilePath)) { - System.err.println("File " + srcFilePathStr + " does not exist."); - return; - } - if (!Files.isRegularFile(srcFilePath)) { - System.err.println("File " + srcFilePathStr + " not a file."); - return; - } - - System.out.println("Putting object file in S3 bucket "); - PutObjectRequest req = new PutObjectRequest(bucketName, objectKey, srcFilePath.toFile()); - if (attributes != null) { - s3.putObject(req, new CreateKeysRequest.Key("", 1, attributes)); - } else { - s3.putObject(req); - } - } - - static void getString(String bucketName, String objectKey, AmazonS3Encryption s3) { - System.out.println("Getting object as string from specified S3 bucket"); - S3Object obj = s3.getObject(bucketName, objectKey); - try { - System.out.println(IOUtils.toString(obj.getObjectContent())); - } catch (IOException e) { - System.err.println("IOException reading content from S3 object"); - } - } - - static void getFile(String bucketName, String objectKey, String destination, - AmazonS3Encryption s3) { - String destFilePathStr = getCanonicalPathString(destination); - - if ((destFilePathStr == null) || (destFilePathStr.isEmpty())) { - System.err.println("No filepath specified"); - return; - } - - Path destFilePath = null; - - // Sandbox within user home - if ((useSandbox) && (!destFilePathStr.startsWith(HOME))) { - System.err.println("Filepath outside of user home"); - return; - } - - destFilePath = Paths.get(destFilePathStr); - - // Check if file already exists but is not a file (e.g. don't try to overwrite a directory) - if ((Files.exists(destFilePath)) && (!Files.isRegularFile(destFilePath))) { - System.err.println("File " + destFilePathStr + " not a file"); - return; - } - - try { - // Safe to delete existing file - Files.deleteIfExists(destFilePath); - } catch (IOException e) { - System.err.println("IOException delete destination: " + destFilePathStr); - return; - } - - System.out.println("Getting object from S3 bucket"); - S3Object obj = s3.getObject(bucketName, objectKey); - try { - Files.copy(obj.getObjectContent(), destFilePath); - } catch (IOException e) { - System.err.println("IOException with destination: " + destFilePathStr); - return; - } - } - - static void putMultipart(String bucketName, String objectKey, String filePath, int partSize, - IonicS3EncryptionClient s3, KeyAttributesMap attributes) { - - long partSizeBytes = ((long) partSize * 1024 * 1024); - - if (partSizeBytes < (5L * 1024 * 1024)) { - System.out.println("Part size must be 5(mb) or greater"); - return; - } - - String srcFilePathStr = getCanonicalPathString(filePath); - - if ((srcFilePathStr == null) || (srcFilePathStr.isEmpty())) { - System.err.println("No filepath specified"); - return; - } - - // Sandbox within user home - if ((useSandbox) && (!srcFilePathStr.startsWith(HOME))) { - System.out.println("Filepath outside of user home"); - return; - } - - Path srcFilePath = Paths.get(srcFilePathStr); - - if (!Files.exists(srcFilePath)) { - System.err.println("Source file does not exist"); - return; - } - if (!Files.isRegularFile(srcFilePath)) { - System.err.println("Source file is not a file"); - return; - } - - File file = srcFilePath.toFile(); - - List partETags = new ArrayList(); - - long contentLength = file.length(); - if (partSizeBytes <= 0) { - System.out.println("Invalid partition size"); - return; - } - long totalChunks = contentLength / partSizeBytes; - - if (contentLength % partSizeBytes != 0) { - totalChunks++; - } - - if (totalChunks == 0L) { - System.out.println("No Multipart upload of an empty file"); - return; - } - if (totalChunks == 1L) { - putFile(bucketName, objectKey, srcFilePathStr, s3, attributes); - return; - } - // Note: Number of parts for Amazon S3 must be between 1 and 10000 - // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html - if (totalChunks > (long) MAX_AWS_SDK_CHUNKS) { - System.out.println( - "File must be limited to " + MAX_AWS_SDK_CHUNKS + " partitions for multipart upload"); - return; - } - int totalChunksInt = (int) totalChunks; - - System.out.println("Initating Multipart Upload"); - System.out.println("With chunk size of " + partSizeBytes + "bytes"); - System.out.println(totalChunksInt + " parts to upload"); - - // Step 1: Initiate multipart upload request - InitiateMultipartUploadRequest req = - new InitiateMultipartUploadRequest(bucketName, objectKey, null); - InitiateMultipartUploadResult res; - if (attributes != null) { - res = s3.initiateMultipartUpload(req, new CreateKeysRequest.Key("", 1, attributes)); - } else { - res = s3.initiateMultipartUpload(req); - } - try { - // Step 2: Upload parts. - long filePosition = 0L; - for (int i = 1; i <= Math.min(MAX_AWS_SDK_CHUNKS, totalChunksInt); i++) { - if (filePosition >= contentLength) { - break; - } - // Last part can be less than 5 MB. Adjust part size. - partSizeBytes = Math.min(partSizeBytes, (contentLength - filePosition)); - - boolean isLast = i == totalChunksInt; - - // Create request to upload a part. - UploadPartRequest uploadRequest = - new UploadPartRequest().withBucketName(bucketName).withKey(objectKey) - .withUploadId(res.getUploadId()).withPartNumber(i).withFileOffset(filePosition) - .withFile(file).withPartSize(partSizeBytes).withLastPart(isLast); - - // Upload part and add response to our list. - System.out.println("Uploading Part #" + i + " of " + totalChunksInt); - partETags.add(s3.uploadPart(uploadRequest).getPartETag()); - if (isLast) { - System.out.println("Uploaded last part"); - } - - filePosition += partSizeBytes; - } - - // Step 3: Complete. - CompleteMultipartUploadRequest compRequest = - new CompleteMultipartUploadRequest(bucketName, objectKey, res.getUploadId(), partETags); - - s3.completeMultipartUpload(compRequest); - System.out.println("Upload Complete"); - } catch (Exception e) { - System.err.println("Exception Occured: " + e.getMessage()); - System.out.println("Aborting Upload"); - s3.abortMultipartUpload( - new AbortMultipartUploadRequest(bucketName, objectKey, res.getUploadId())); - } - } - - static IonicS3EncryptionClient setUp() - throws IOException, InvalidPathException, IllegalArgumentException { - // Set up IonicEncryptionMaterialsProvider and provide to constructor of IonicS3EncryptionClient - IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); - IonicEncryptionMaterialsProvider.setIonicMetadataMap(S3SampleApp.getMetadataMap()); - - // Load a plain-text device profile (SEP) from disk - DeviceProfilePersistorPlainText ptPersistor = new DeviceProfilePersistorPlainText(); - - String sProfilePath = - Paths.get(HOME + "/.ionicsecurity/profiles.pt").toFile().getCanonicalPath(); - try { - ptPersistor.setFilePath(sProfilePath); - } catch (IonicException e) { - System.err.println("Error: Can't set the profile persistor path"); - return null; - } - iemp.setPersistor(ptPersistor); - - return (IonicS3EncryptionClient)IonicS3EncryptionClientBuilder.standard().withEncryptionMaterials(iemp).build(); - } - - public static void main(String[] args) - throws IOException, InvalidPathException, IllegalArgumentException { - - Action action = null; - boolean mFlag = false; - int actionArg = 0; - int mFlagArg = 1; - int bucketNameArg = 1; - int objectKeyArg = 2; - int objectContentArg = 3; - int filePathArg = 3; - int attributesArg = 4; - int partSizeArg = 4; - - // Command Line Processing - if (args.length > actionArg) { - // Determine Action (e.g. getString) - for (Action a : Action.values()) { - if (a.str.equals(args[actionArg])) { - action = a; - break; - } - } - } else { - usage(); - return; - } - - String bucketName = null; - String objectKey = null; - String objectContent = null; - String filePath = null; - KeyAttributesMap attributes = null; - - IonicS3EncryptionClient s3; - - switch (action) { - case PUTFILE: - if (args.length > filePathArg) { - bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); - - objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); - - filePath = new String(args[filePathArg].getBytes(StringUtils.UTF8)); - - // Optional: parse any attributes - if (args.length > attributesArg) { - attributes = parseAttributes(args[attributesArg]); - if (attributes == null) { - return; - } - } - - s3 = setUp(); - if (s3 != null) { - putFile(bucketName, objectKey, filePath, s3, attributes); - } - } else { - usage(); - } - break; - case PUTSTRING: - if (args.length > filePathArg) { - bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); - - objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); - - objectContent = new String(args[objectContentArg].getBytes(StringUtils.UTF8)); - - // Optional: parse any attributes - if (args.length > attributesArg) { - attributes = parseAttributes(args[attributesArg]); - if (attributes == null) { - return; - } - } - - s3 = setUp(); - if (s3 != null) { - putString(bucketName, objectKey, objectContent, s3, attributes); - } - } else { - usage(); - } - break; - - case PUTMULTIPART: - attributesArg++; // increment to account for addition of partSize arg - if (args.length > partSizeArg) { - bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); - - objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); - - filePath = new String(args[filePathArg].getBytes(StringUtils.UTF8)); - - // Parse the size of each part being uploaded in MB - // Note: parts must be at least 5MB (except for last part which may be smaller) - // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html - int partSizeMB = Math.max(Integer.parseInt(args[partSizeArg]), 0); - if ((partSizeMB == 0) || (partSizeMB < 5)) { - System.out.println("partsize_mb must be a positive integer value 5 or greater"); - usage(); - return; - } - - // Optional: parse any attributes - if (args.length > attributesArg) { - attributes = parseAttributes(args[attributesArg]); - if (attributes == null) { - return; - } - } - - s3 = setUp(); - if (s3 != null) { - putMultipart(bucketName, objectKey, filePath, partSizeMB, s3, attributes); - } - } else { - usage(); - } - - break; - case GETSTRING: - if (args.length > objectKeyArg) { - bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); - - objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); - - s3 = setUp(); - if (s3 != null) { - getString(bucketName, objectKey, s3); - } - } else { - usage(); - } - - break; - case GETFILE: - if (args.length > filePathArg) { - bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); - - objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); - - filePath = new String(args[filePathArg].getBytes(StringUtils.UTF8)); - - s3 = setUp(); - if (s3 != null) { - getFile(bucketName, objectKey, filePath, s3); - } - } else { - usage(); - } - - break; - case VERSION: - System.out.println(Version.getFullVersion()); - break; - default: - usage(); - break; - } - } - - private static void usage() { - System.out.println("Usage: prog command> | command> | version"); - System.out.println("put commands:"); - System.out - .println("\tNOTE: for these commands is a comma delimited list of key:value1,value2... "); - System.out.println("\t\tEx: \"attribute1:val1:val2,attribute2:val3\" -> [attribute1:val1,val2],[attribute2:val3]"); - System.out.println(""); - System.out.println("\tputString []"); - System.out.println("\tputFile []"); - System.out.println("\tputMultipart []"); - System.out.println("get commands:"); - System.out.println("\tgetFile "); - System.out.println("\tgetString "); - } - - public static KeyAttributesMap parseAttributes(String str) { - KeyAttributesMap ret = new KeyAttributesMap(); - String[] pairs = str.split(","); - for (String pair : pairs){ - String[] tuples = pair.split(":"); - ArrayList values = new ArrayList(); - for(int i = 1; i < tuples.length; i++ ) { - values.add(tuples[i]); - } - ret.put(tuples[0],values); - } - return ret; - } - - public static MetadataMap getMetadataMap() { - MetadataMap mApplicationMetadata = new MetadataMap(); - mApplicationMetadata.set("ionic-application-name", "IonicS3Example"); - mApplicationMetadata.set("ionic-application-version", Version.getFullVersion()); - mApplicationMetadata.set("ionic-client-type", "IPCS S3 Java"); - mApplicationMetadata.set("ionic-client-version", Version.getFullVersion()); - - return mApplicationMetadata; - } - - public static String getCanonicalPathString(String originalPath) { - String canonicalPathStr = null; - - try { - canonicalPathStr = Paths.get(originalPath).toFile().getCanonicalPath(); - } catch (NullPointerException e) { - System.err.println("Missing original pathname"); - } catch (IOException e) { - System.err.println("Path IOError"); - } - return canonicalPathStr; - } -} diff --git a/awss3examples/src/main/java/com/ionicsecurity/examples/S3TransferManagerSample.java b/awss3examples/src/main/java/com/ionicsecurity/examples/S3TransferManagerSample.java deleted file mode 100644 index 7745c69..0000000 --- a/awss3examples/src/main/java/com/ionicsecurity/examples/S3TransferManagerSample.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * (c) 2017-2018 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as - * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy - * (https://www.ionic.com/privacy-notice/). - */ - -package com.ionicsecurity.examples; - -import java.io.IOException; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3Encryption; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.transfer.Download; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; -import com.ionicsecurity.ipcs.awss3.IonicEncryptionMaterialsProvider; -import com.ionicsecurity.ipcs.awss3.IonicS3EncryptionClientBuilder; -import com.ionicsecurity.ipcs.awss3.Version; -import com.ionic.sdk.agent.data.MetadataMap; -import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; -import com.ionic.sdk.error.IonicException; - - -public class S3TransferManagerSample { - - public static final boolean useSandbox = true; // if true limit file paths to within user's - // home dir - private static String HOME = System.getProperty("user.home"); - - public static void main(String[] args) { - - if (args.length != 4) { - usage(); - return; - } - - String bucketName = args[1]; - String objectKey = args[2]; - - Path xfrFilePath = null; - try { - xfrFilePath = Paths.get(Paths.get(args[3]).toFile().getCanonicalPath()); - } catch (NullPointerException e) { - System.err.println("Missing source file pathname"); - return; - } catch (IOException e) { - System.err.println("Path IOError"); - return; - } - - // Sandbox within user home - if ((useSandbox) && (!xfrFilePath.startsWith(HOME))) { - System.err.println("Filepath outside of user home"); - return; - } - if (!Files.exists(xfrFilePath)) { - System.err.println("Transfer file does not exist."); - return; - } - if (!Files.isRegularFile(xfrFilePath)) { - System.err.println("Transfer file not a file."); - return; - } - - // Set up IonicEncryptionMaterialsProvider and provide to constructor of IonicS3EncryptionClient - IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); - IonicEncryptionMaterialsProvider.setIonicMetadataMap(S3SampleApp.getMetadataMap()); - - DeviceProfilePersistorPlainText ptPersistor = new DeviceProfilePersistorPlainText(); - try { - String sProfilePath = - Paths.get(HOME + "/.ionicsecurity/profiles.pt").toFile().getCanonicalPath(); - ptPersistor.setFilePath(sProfilePath); - } catch (IOException e) { - System.err.println(e.getMessage()); - return; - } catch (IonicException e) { - System.err.println(e.getMessage()); - return; - } - iemp.setPersistor(ptPersistor); - - AmazonS3Encryption s3 = null; - - try { - s3 = IonicS3EncryptionClientBuilder.standard().withEncryptionMaterials(iemp).build(); - } catch (IllegalArgumentException e) { - System.err.println(e.getMessage()); - return; - } - - TransferManager manager = TransferManagerBuilder.standard().withS3Client(s3).build(); - - if (args[0].equals("put")) { - Upload upload = - manager.upload(new PutObjectRequest(bucketName, objectKey, xfrFilePath.toFile())); - try { - upload.waitForCompletion(); - } catch (AmazonClientException | InterruptedException e) { - System.err.println(e.getMessage()); - } - } else if (args[0].equals("get")) { - Download download = manager.download(bucketName, objectKey, xfrFilePath.toFile()); - try { - download.waitForCompletion(); - } catch (AmazonClientException | InterruptedException e) { - System.err.println(e.getMessage()); - } - } else { - usage(); - } - // Shut down transferManager and underlying S3 client - manager.shutdownNow(true); - return; - } - - private static void usage() { - System.out.println("put "); - System.out.println("get + + 4.0.0 + com.ionic.cloudstorage.samples + ionic-awss3-example + 1.0.0 + jar + https://ionic.com + + UTF-8 + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.5.1 + + 1.8 + 1.8 + true + true + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + package + + shade + + + IonicS3SampleApp + + + com.ionic.cloudstorage.samples.S3SampleApp + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + com.ionic.cloudstorage + ionic-for-aws-java-sdk-s3 + 1.0.0 + + + diff --git a/awss3examples/run.bat b/examples/run.bat similarity index 100% rename from awss3examples/run.bat rename to examples/run.bat diff --git a/awss3examples/run.sh b/examples/run.sh similarity index 100% rename from awss3examples/run.sh rename to examples/run.sh diff --git a/examples/src/main/java/com/ionic/cloudstorage/samples/S3SampleApp.java b/examples/src/main/java/com/ionic/cloudstorage/samples/S3SampleApp.java new file mode 100644 index 0000000..966d5ee --- /dev/null +++ b/examples/src/main/java/com/ionic/cloudstorage/samples/S3SampleApp.java @@ -0,0 +1,540 @@ +/* + * S3SampleApp.java The purpose of this project is to store an object in AWS S3 with client-side + * Ionic protection. This code is an example of what clients would use programmatically to + * incorporate the Ionic platform into their S3 use cases. + * + * (c) 2017-2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as + * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). Derived in part from AWS Sample S3 Project, + * S3Sample.java. + */ + +package com.ionic.cloudstorage.samples; + +import java.io.IOException; +import java.nio.file.InvalidPathException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.util.Map; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.ionic.sdk.agent.data.MetadataMap; +import com.ionic.sdk.agent.key.KeyAttributesMap; +import com.ionic.sdk.agent.request.createkey.CreateKeysRequest; +import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; +import com.ionic.sdk.error.IonicException; +import com.ionic.cloudstorage.awss3.IonicEncryptionMaterialsProvider; +import com.ionic.cloudstorage.awss3.IonicS3EncryptionClient; +import com.ionic.cloudstorage.awss3.IonicS3EncryptionClientBuilder; +import com.ionic.cloudstorage.awss3.Version; +import com.amazonaws.SdkClientException; +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.util.IOUtils; +import com.amazonaws.util.StringUtils; + +public class S3SampleApp { + + enum Action { + GETSTRING("getString"), + GETFILE("getFile"), + PUTSTRING("putString"), + PUTFILE("putFile"), + PUTMULTIPART("putMultipart"), + VERSION("version"),; + + final String str; + + Action(String name) { + this.str = name; + } + } + + public static final int MAX_AWS_SDK_CHUNKS = 10000; // AWS S3 limits multipart uploads to 10000 + // parts + public static final boolean useSandbox = true; // if true limit file paths to within user's home + // dir + private static final String HOME = System.getProperty("user.home"); + + + static void putString(String bucketName, String objectKey, String objectContent, + IonicS3EncryptionClient s3, KeyAttributesMap attributes) { + System.out.println("Putting object as string in specified S3 bucket"); + if (attributes != null) { + s3.putObject(bucketName, objectKey, objectContent, + new CreateKeysRequest.Key("", 1, attributes)); + } else { + s3.putObject(bucketName, objectKey, objectContent); + } + } + + static void putFile(String bucketName, String objectKey, String filePath, + IonicS3EncryptionClient s3, KeyAttributesMap attributes) { + + String srcFilePathStr = getCanonicalPathString(filePath); + + if ((srcFilePathStr == null) || (srcFilePathStr.isEmpty())) { + System.err.println("No filepath specified"); + return; + } + + // Sandbox within user home + if ((useSandbox) && (!srcFilePathStr.startsWith(HOME))) { + System.err.println("Filepath outside of user home"); + return; + } + + Path srcFilePath = Paths.get(srcFilePathStr); + + if (!Files.exists(srcFilePath)) { + System.err.println("File " + srcFilePathStr + " does not exist."); + return; + } + if (!Files.isRegularFile(srcFilePath)) { + System.err.println("File " + srcFilePathStr + " not a file."); + return; + } + + System.out.println("Putting object file in S3 bucket "); + PutObjectRequest req = new PutObjectRequest(bucketName, objectKey, srcFilePath.toFile()); + if (attributes != null) { + s3.putObject(req, new CreateKeysRequest.Key("", 1, attributes)); + } else { + s3.putObject(req); + } + } + + static void getString(String bucketName, String objectKey, AmazonS3Encryption s3) { + System.out.println("Getting object as string from specified S3 bucket"); + S3Object obj = s3.getObject(bucketName, objectKey); + try { + System.out.println(IOUtils.toString(obj.getObjectContent())); + } catch (IOException e) { + System.err.println("IOException reading content from S3 object"); + } + } + + static void getFile(String bucketName, String objectKey, String destination, + AmazonS3Encryption s3) { + String destFilePathStr = getCanonicalPathString(destination); + + if ((destFilePathStr == null) || (destFilePathStr.isEmpty())) { + System.err.println("No filepath specified"); + return; + } + + Path destFilePath = null; + + // Sandbox within user home + if ((useSandbox) && (!destFilePathStr.startsWith(HOME))) { + System.err.println("Filepath outside of user home"); + return; + } + + destFilePath = Paths.get(destFilePathStr); + + // Check if file already exists but is not a file (e.g. don't try to overwrite a directory) + if ((Files.exists(destFilePath)) && (!Files.isRegularFile(destFilePath))) { + System.err.println("File " + destFilePathStr + " not a file"); + return; + } + + try { + // Safe to delete existing file + Files.deleteIfExists(destFilePath); + } catch (IOException e) { + System.err.println("IOException delete destination: " + destFilePathStr); + return; + } + + System.out.println("Getting object from S3 bucket"); + S3Object obj = s3.getObject(bucketName, objectKey); + try { + Files.copy(obj.getObjectContent(), destFilePath); + } catch (IOException e) { + System.err.println("IOException with destination: " + destFilePathStr); + return; + } + } + + static void putMultipart(String bucketName, String objectKey, String filePath, int partSize, + IonicS3EncryptionClient s3, KeyAttributesMap attributes) { + + long partSizeBytes = ((long) partSize * 1024 * 1024); + + if (partSizeBytes < (5L * 1024 * 1024)) { + System.out.println("Part size must be 5(mb) or greater"); + return; + } + + String srcFilePathStr = getCanonicalPathString(filePath); + + if ((srcFilePathStr == null) || (srcFilePathStr.isEmpty())) { + System.err.println("No filepath specified"); + return; + } + + // Sandbox within user home + if ((useSandbox) && (!srcFilePathStr.startsWith(HOME))) { + System.out.println("Filepath outside of user home"); + return; + } + + Path srcFilePath = Paths.get(srcFilePathStr); + + if (!Files.exists(srcFilePath)) { + System.err.println("Source file does not exist"); + return; + } + if (!Files.isRegularFile(srcFilePath)) { + System.err.println("Source file is not a file"); + return; + } + + File file = srcFilePath.toFile(); + + List partETags = new ArrayList(); + + long contentLength = file.length(); + if (partSizeBytes <= 0) { + System.out.println("Invalid partition size"); + return; + } + long totalChunks = contentLength / partSizeBytes; + + if (contentLength % partSizeBytes != 0) { + totalChunks++; + } + + if (totalChunks == 0L) { + System.out.println("No Multipart upload of an empty file"); + return; + } + if (totalChunks == 1L) { + putFile(bucketName, objectKey, srcFilePathStr, s3, attributes); + return; + } + // Note: Number of parts for Amazon S3 must be between 1 and 10000 + // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + if (totalChunks > (long) MAX_AWS_SDK_CHUNKS) { + System.out.println("File must be limited to " + MAX_AWS_SDK_CHUNKS + + " partitions for multipart upload"); + return; + } + int totalChunksInt = (int) totalChunks; + + System.out.println("Initating Multipart Upload"); + System.out.println("With chunk size of " + partSizeBytes + "bytes"); + System.out.println(totalChunksInt + " parts to upload"); + + // Step 1: Initiate multipart upload request + InitiateMultipartUploadRequest req = + new InitiateMultipartUploadRequest(bucketName, objectKey, null); + InitiateMultipartUploadResult res; + if (attributes != null) { + res = s3.initiateMultipartUpload(req, new CreateKeysRequest.Key("", 1, attributes)); + } else { + res = s3.initiateMultipartUpload(req); + } + try { + // Step 2: Upload parts. + long filePosition = 0L; + for (int i = 1; i <= Math.min(MAX_AWS_SDK_CHUNKS, totalChunksInt); i++) { + if (filePosition >= contentLength) { + break; + } + // Last part can be less than 5 MB. Adjust part size. + partSizeBytes = Math.min(partSizeBytes, (contentLength - filePosition)); + + boolean isLast = i == totalChunksInt; + + // Create request to upload a part. + UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucketName) + .withKey(objectKey).withUploadId(res.getUploadId()).withPartNumber(i) + .withFileOffset(filePosition).withFile(file).withPartSize(partSizeBytes) + .withLastPart(isLast); + + // Upload part and add response to our list. + System.out.println("Uploading Part #" + i + " of " + totalChunksInt); + partETags.add(s3.uploadPart(uploadRequest).getPartETag()); + if (isLast) { + System.out.println("Uploaded last part"); + } + + filePosition += partSizeBytes; + } + + // Step 3: Complete. + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest( + bucketName, objectKey, res.getUploadId(), partETags); + + s3.completeMultipartUpload(compRequest); + System.out.println("Upload Complete"); + } catch (Exception e) { + System.err.println("Exception Occured: " + e.getMessage()); + System.out.println("Aborting Upload"); + s3.abortMultipartUpload( + new AbortMultipartUploadRequest(bucketName, objectKey, res.getUploadId())); + } + } + + static IonicS3EncryptionClient setUp() + throws IOException, InvalidPathException, IllegalArgumentException { + // Set up IonicEncryptionMaterialsProvider and provide to constructor of + // IonicS3EncryptionClient + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); + iemp.setIonicMetadataMap(S3SampleApp.getMetadataMap()); + + // Load a plain-text device profile (SEP) from disk + DeviceProfilePersistorPlainText ptPersistor = new DeviceProfilePersistorPlainText(); + + String sProfilePath = + Paths.get(HOME + "/.ionicsecurity/profiles.pt").toFile().getCanonicalPath(); + try { + ptPersistor.setFilePath(sProfilePath); + iemp.setPersistor(ptPersistor); + } catch (IonicException e) { + System.err.println(e.getMessage()); + return null; + } + + return (IonicS3EncryptionClient) IonicS3EncryptionClientBuilder.standard() + .withEncryptionMaterials(iemp).build(); + } + + public static void main(String[] args) + throws IOException, InvalidPathException, IllegalArgumentException { + + Action action = null; + boolean mFlag = false; + int actionArg = 0; + int mFlagArg = 1; + int bucketNameArg = 1; + int objectKeyArg = 2; + int objectContentArg = 3; + int filePathArg = 3; + int attributesArg = 4; + int partSizeArg = 4; + + // Command Line Processing + if (args.length > actionArg) { + // Determine Action (e.g. getString) + for (Action a : Action.values()) { + if (a.str.equals(args[actionArg])) { + action = a; + break; + } + } + if (action == null) { + usage(); + return; + } + } else { + usage(); + return; + } + + String bucketName = null; + String objectKey = null; + String objectContent = null; + String filePath = null; + KeyAttributesMap attributes = null; + + IonicS3EncryptionClient s3; + + switch (action) { + case PUTFILE: + if (args.length > filePathArg) { + bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); + + objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); + + filePath = new String(args[filePathArg].getBytes(StringUtils.UTF8)); + + // Optional: parse any attributes + if (args.length > attributesArg) { + attributes = parseAttributes(args[attributesArg]); + if (attributes == null) { + return; + } + } + + s3 = setUp(); + if (s3 != null) { + putFile(bucketName, objectKey, filePath, s3, attributes); + } + } else { + usage(); + } + break; + case PUTSTRING: + if (args.length > filePathArg) { + bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); + + objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); + + objectContent = new String(args[objectContentArg].getBytes(StringUtils.UTF8)); + + // Optional: parse any attributes + if (args.length > attributesArg) { + attributes = parseAttributes(args[attributesArg]); + if (attributes == null) { + return; + } + } + + s3 = setUp(); + if (s3 != null) { + putString(bucketName, objectKey, objectContent, s3, attributes); + } + } else { + usage(); + } + break; + + case PUTMULTIPART: + attributesArg++; // increment to account for addition of partSize arg + if (args.length > partSizeArg) { + bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); + + objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); + + filePath = new String(args[filePathArg].getBytes(StringUtils.UTF8)); + + // Parse the size of each part being uploaded in MB + // Note: parts must be at least 5MB (except for last part which may be smaller) + // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + int partSizeMB = Math.max(Integer.parseInt(args[partSizeArg]), 0); + if ((partSizeMB == 0) || (partSizeMB < 5)) { + System.out.println( + "partsize_mb must be a positive integer value 5 or greater"); + usage(); + return; + } + + // Optional: parse any attributes + if (args.length > attributesArg) { + attributes = parseAttributes(args[attributesArg]); + if (attributes == null) { + return; + } + } + + s3 = setUp(); + if (s3 != null) { + putMultipart(bucketName, objectKey, filePath, partSizeMB, s3, attributes); + } + } else { + usage(); + } + + break; + case GETSTRING: + if (args.length > objectKeyArg) { + bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); + + objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); + + s3 = setUp(); + if (s3 != null) { + getString(bucketName, objectKey, s3); + } + } else { + usage(); + } + + break; + case GETFILE: + if (args.length > filePathArg) { + bucketName = new String(args[bucketNameArg].getBytes(StringUtils.UTF8)); + + objectKey = new String(args[objectKeyArg].getBytes(StringUtils.UTF8)); + + filePath = new String(args[filePathArg].getBytes(StringUtils.UTF8)); + + s3 = setUp(); + if (s3 != null) { + getFile(bucketName, objectKey, filePath, s3); + } + } else { + usage(); + } + + break; + case VERSION: + System.out.println(Version.getFullVersion()); + break; + default: + usage(); + break; + } + } + + private static void usage() { + System.out.println("Usage: prog command> | command> | version"); + System.out.println("put commands:"); + System.out.println("\tNOTE: for these commands is a list of comma delimited tuples " + + "with each tuple composed of a key followed by a colon delimited list of values"); + System.out.println( "\t\t:[:]…[,:[:]…]…"); + System.out.println( "\t\tExample: attribute1:value1:value2,attribute2:value3"); + System.out.println("\tputString []"); + System.out.println("\tputFile []"); + System.out.println( + "\tputMultipart []"); + System.out.println("get commands:"); + System.out.println("\tgetFile "); + System.out.println("\tgetString "); + } + + public static KeyAttributesMap parseAttributes(String str) { + KeyAttributesMap ret = new KeyAttributesMap(); + String[] pairs = str.split(","); + for (String pair : pairs) { + String[] tuples = pair.split(":"); + ArrayList values = new ArrayList(); + for (int i = 1; i < tuples.length; i++) { + values.add(tuples[i]); + } + ret.put(tuples[0], values); + } + return ret; + } + + public static MetadataMap getMetadataMap() { + MetadataMap mApplicationMetadata = new MetadataMap(); + mApplicationMetadata.set("ionic-application-name", "IonicS3Example"); + mApplicationMetadata.set("ionic-application-version", Version.getFullVersion()); + mApplicationMetadata.set("ionic-client-type", "IPCS S3 Java"); + mApplicationMetadata.set("ionic-client-version", Version.getFullVersion()); + + return mApplicationMetadata; + } + + public static String getCanonicalPathString(String originalPath) { + String canonicalPathStr = null; + + try { + canonicalPathStr = Paths.get(originalPath).toFile().getCanonicalPath(); + } catch (NullPointerException e) { + System.err.println("Missing original pathname"); + } catch (IOException e) { + System.err.println("Path IOError"); + } + return canonicalPathStr; + } +} diff --git a/examples/src/main/java/com/ionic/cloudstorage/samples/S3TransferManagerSample.java b/examples/src/main/java/com/ionic/cloudstorage/samples/S3TransferManagerSample.java new file mode 100644 index 0000000..8e4bbcb --- /dev/null +++ b/examples/src/main/java/com/ionic/cloudstorage/samples/S3TransferManagerSample.java @@ -0,0 +1,137 @@ +/* + * (c) 2017-2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as + * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.samples; + +import java.io.IOException; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.Download; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; +import com.ionic.cloudstorage.awss3.IonicEncryptionMaterialsProvider; +import com.ionic.cloudstorage.awss3.IonicS3EncryptionClientBuilder; +import com.ionic.cloudstorage.awss3.Version; +import com.ionic.sdk.agent.data.MetadataMap; +import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; +import com.ionic.sdk.error.IonicException; + + +public class S3TransferManagerSample { + + public static final boolean useSandbox = true; // if true limit file paths to within user's + // home dir + private static String HOME = System.getProperty("user.home"); + + public static void main(String[] args) { + + if (args.length != 4) { + usage(); + return; + } + + String bucketName = args[1]; + String objectKey = args[2]; + + Path xfrFilePath = null; + try { + xfrFilePath = Paths.get(Paths.get(args[3]).toFile().getCanonicalPath()); + } catch (NullPointerException e) { + System.err.println("Missing source file pathname"); + return; + } catch (IOException e) { + System.err.println("Path IOError"); + return; + } + + // Sandbox within user home + if ((useSandbox) && (!xfrFilePath.startsWith(HOME))) { + System.err.println("Filepath outside of user home"); + return; + } + if (!Files.exists(xfrFilePath)) { + System.err.println("Transfer file does not exist."); + return; + } + if (!Files.isRegularFile(xfrFilePath)) { + System.err.println("Transfer file not a file."); + return; + } + + // Set up IonicEncryptionMaterialsProvider and provide to constructor of + // IonicS3EncryptionClient + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); + iemp.setIonicMetadataMap(S3SampleApp.getMetadataMap()); + + DeviceProfilePersistorPlainText ptPersistor = new DeviceProfilePersistorPlainText(); + try { + String sProfilePath = + Paths.get(HOME + "/.ionicsecurity/profiles.pt").toFile().getCanonicalPath(); + ptPersistor.setFilePath(sProfilePath); + iemp.setPersistor(ptPersistor); + } catch (IOException e) { + System.err.println(e.getMessage()); + return; + } catch (IonicException e) { + System.err.println(e.getMessage()); + return; + } + + AmazonS3Encryption s3 = null; + + try { + s3 = IonicS3EncryptionClientBuilder.standard().withEncryptionMaterials(iemp).build(); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + return; + } + + TransferManager manager = TransferManagerBuilder.standard().withS3Client(s3).build(); + + if (args[0].equals("put")) { + Upload upload = manager + .upload(new PutObjectRequest(bucketName, objectKey, xfrFilePath.toFile())); + try { + upload.waitForCompletion(); + } catch (AmazonClientException | InterruptedException e) { + System.err.println(e.getMessage()); + } + } else if (args[0].equals("get")) { + Download download = manager.download(bucketName, objectKey, xfrFilePath.toFile()); + try { + download.waitForCompletion(); + } catch (AmazonClientException | InterruptedException e) { + System.err.println(e.getMessage()); + } + } else { + usage(); + } + // Shut down transferManager and underlying S3 client + manager.shutdownNow(true); + return; + } + + private static void usage() { + System.out.println("put "); + System.out.println("get - - 4.0.0 + + 4.0.0 - com.ionicsecurity.ipcs.awss3 - ionics3 - jar - 0.9.0 + com.ionic.cloudstorage + ionic-for-aws-java-sdk-s3 + jar + 1.0.0 - ionics3 - Ionic Protect for AWS S3 Cloud Storage offers a simple way for developers building atop AWS S3’s Java SDK to invoke Ionic’s protection and policy functionality as data moves to and from the cloud storage. This addresses use cases such as migration from on-prem storage solutions, protecting data across multi-region or multi-cloud envrionments, applying granular cryptographic control, and more. - https://dev.ionic.com/integrations/ipcs_s3.html + ionic-for-aws-java-sdk-s3 + Ionic Protect for AWS S3 Cloud Storage offers a simple way for developers building atop AWS S3’s Java SDK to invoke Ionic’s protection and policy functionality as data moves to and from the cloud storage. This addresses use cases such as migration from on-prem storage solutions, protecting data across multi-region or multi-cloud envrionments, applying granular cryptographic control, and more. + https://dev.ionic.com/integrations/aws-s3 - - - License Agreement for Ionic Resources - https://github.com/IonicDev/ipcs-s3/blob/master/LICENSE.md - - + + + License Agreement for Ionic Resources + https://github.com/IonicDev/ipcs-s3/blob/master/LICENSE.md + + - - - Ionic Security - dev@ionic.com - Ionic Security - https://www.ionic.com - - + + + Ionic Security + dev@ionic.com + Ionic Security + https://www.ionic.com + + - - scm:git:git://github.com/IonicDev/ipcs-s3.git - scm:git:ssh://github.com/IonicDev/ipcs-s3.git - https://github.com/IonicDev/ipcs-s3/tree/master - + + scm:git:git://github.com/IonicDev/ipcs-s3.git + scm:git:ssh://github.com/IonicDev/ipcs-s3.git + https://github.com/IonicDev/ipcs-s3/tree/master + - - LOCAL - UTF-8 - true - + + LOCAL + UTF-8 + - - - - org.apache.maven.plugins - maven-compiler-plugin - 2.5.1 - - 1.8 - 1.8 - true - true - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.19.1 - - ${skipTests} - once - target - - - - org.codehaus.mojo - templating-maven-plugin - 1.0.0 - - - filter-src - - filter-sources - - - - - - org.apache.maven.plugins - 3.0.0 - maven-javadoc-plugin - - true - - com.ionic:ionic-sdk - com.amazonaws:aws-java-sdk-s3 - - com.ionic.sdk.core.*:com.ionic.sdk.crypto*:com.ionic.sdk.cipher*:com.ionic.sdk.httpclient*:com.ionic.sdk.agent.cipher*:com.amazonaws.auth*:com.amazonaws.services.s3.model.*:com.amazonaws.services.s3.i*:com.amazonaws.services.s3.t* - - -Xdoclint:none - - - - - + + + org.glassfish + javax.json + 1.0.4 + + + org.bouncycastle + bcprov-jdk15on + 1.56 + + + junit + junit + 4.12 + test + + + com.amazonaws + aws-java-sdk-s3 + 1.11.458 + + + com.amazonaws + aws-java-sdk-iam + 1.11.458 + + + com.ionic + ionic-sdk + 2.4.0 + + + commons-io + commons-io + 2.6 + + + javax.xml.bind + jaxb-api + 2.3.1 + + - - - junit - junit - 4.12 - test - - - com.amazonaws - aws-java-sdk-s3 - 1.11.458 - - - com.amazonaws - aws-java-sdk-iam - 1.11.458 - - - com.ionic - ionic-sdk - 2.3.0 - - - commons-io - commons-io - 1.3.2 - - - javax.xml.bind - jaxb-api - 2.3.1 - - + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 1.8 + 1.8 + true + true + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M3 + + + + listener + com.ionic.junit.listen.SkipFailListener + + + once + target + + + + org.codehaus.mojo + templating-maven-plugin + 1.0.0 + + + filter-src + + filter-sources + + + + + + org.apache.maven.plugins + 3.1.0 + maven-javadoc-plugin + + true + + com.ionic:ionic-sdk + com.amazonaws:aws-java-sdk-s3 + + com.ionic.sdk.core.*:com.ionic.sdk.crypto*:com.ionic.sdk.cipher*:com.ionic.sdk.httpclient*:com.ionic.sdk.agent.cipher*:com.amazonaws.auth*:com.amazonaws.services.s3.model.*:com.amazonaws.services.s3.i*:com.amazonaws.services.s3.t* + + -Xdoclint:none + + + + + diff --git a/src/main/java-templates/Version.java b/src/main/java-templates/Version.java index 1c94747..d5a1667 100644 --- a/src/main/java-templates/Version.java +++ b/src/main/java-templates/Version.java @@ -1,8 +1,8 @@ -package com.ionicsecurity.ipcs.awss3; +package com.ionic.cloudstorage.awss3; public final class Version { - private static final String BUILD = "${build}"; + private static final String BUILD = "${build}"; private static final String VERSION = "${project.version}"; private static final String GROUPID = "${project.groupId}"; private static final String ARTIFACTID = "${project.artifactId}"; @@ -10,33 +10,32 @@ public final class Version { public static String getMajorVersion() { return VERSION.split("\\.")[0]; } - + public static String getMinorVersion() { return VERSION.split("\\.")[1]; } - + public static String getPatchVersion() { return VERSION.split("\\.")[2]; } - - public static String getBuild() - { + + public static String getBuild() { return BUILD; } - + public static String getGroupId() { return GROUPID; } - + public static String getArtifactId() { return ARTIFACTID; } - + public static String getVersion() { return VERSION; } - + public static String getFullVersion() { return VERSION + "-" + BUILD; } -} \ No newline at end of file +} diff --git a/src/main/java/com/ionic/cloudstorage/awss3/IonicAgentFactory.java b/src/main/java/com/ionic/cloudstorage/awss3/IonicAgentFactory.java new file mode 100644 index 0000000..7bc9fd1 --- /dev/null +++ b/src/main/java/com/ionic/cloudstorage/awss3/IonicAgentFactory.java @@ -0,0 +1,86 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as + * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import com.ionic.sdk.agent.Agent; +import com.ionic.sdk.device.profile.DeviceProfile; +import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase; +import com.ionic.sdk.error.AgentErrorModuleConstants; +import com.ionic.sdk.error.IonicException; +import com.ionic.sdk.agent.data.MetadataMap; + +/** + * AgentFactory provides a means of acquiring initialized Agents in a thread safe manner that + * minimizes the overhead of constructing and initiating Agents on demand. + */ +public class IonicAgentFactory { + + private MetadataMap metadataMap = new MetadataMap(); + + private DeviceProfile profile; + + /** + * getAgent() creates an Agent and initializes it with the given active profile and metadataMap + * + * @return a {@link com.ionic.sdk.agent.Agent} object. + * @throws com.ionic.sdk.error.IonicException if the + * {@link com.ionic.sdk.device.profile.DeviceProfile active profile} + * is unset. + */ + public Agent getAgent() throws IonicException{ + if (profile == null) { + throw new IonicException(AgentErrorModuleConstants.ISAGENT_NO_DEVICE_PROFILE); + } + Agent agent = new Agent(); + agent.initializeWithoutProfiles(); + agent.addProfile(profile, true); + agent.setMetadata(this.getMetadataMap()); + return agent; + } + + /** + * setProfile() sets the profile to use when making new agents based on the active profile in + * the given persistor. + * + * @param persistor a {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase} + * object. + * @throws com.ionic.sdk.error.IonicException if the + * {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase Persistor} + * is unset or does not contain an active profile. + */ + public void setActiveProfile(DeviceProfilePersistorBase persistor) throws IonicException { + Agent agent = new Agent(); + agent.initialize(persistor); + if (agent.hasActiveProfile() == false) { + throw new IonicException(AgentErrorModuleConstants.ISAGENT_NO_DEVICE_PROFILE); + } + profile = agent.getActiveProfile(); + } + + /** + * setMetadataMap() sets the MetadataMap to be used when generating new Agents + * + * @param map a {@link com.ionic.sdk.agent.data.MetadataMap} object. + */ + void setMetadataMap(MetadataMap map) { + MetadataMap newMap = new MetadataMap(); + newMap.putAll(map); + this.metadataMap = newMap; + } + + /** + * getMetadataMap() gets a copy of Agent.MetadataMap + * + * @return a {@link com.ionic.sdk.agent.data.MetadataMap} object. + */ + MetadataMap getMetadataMap() { + MetadataMap map = new MetadataMap(); + map.putAll(this.metadataMap); + return map; + } + +} diff --git a/src/main/java/com/ionicsecurity/ipcs/awss3/IonicEncryptionMaterialsProvider.java b/src/main/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProvider.java similarity index 62% rename from src/main/java/com/ionicsecurity/ipcs/awss3/IonicEncryptionMaterialsProvider.java rename to src/main/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProvider.java index 97a3643..6ce385d 100644 --- a/src/main/java/com/ionicsecurity/ipcs/awss3/IonicEncryptionMaterialsProvider.java +++ b/src/main/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProvider.java @@ -1,27 +1,22 @@ /* - * (c) 2017-2018 Ionic Security Inc. - * By using this code, I agree to the LICENSE included, as well as the - * Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy (https://www.ionic.com/privacy-notice/). + * (c) 2017-2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as + * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). */ -package com.ionicsecurity.ipcs.awss3; +package com.ionic.cloudstorage.awss3; import java.io.IOException; import java.nio.file.Paths; import java.nio.file.InvalidPathException; - import java.util.ArrayList; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.HashMap; -import java.util.Iterator; -import java.util.List; +import java.util.Map; import java.util.UUID; - import javax.crypto.SecretKey; import javax.crypto.spec.SecretKeySpec; - import com.ionic.sdk.agent.Agent; -import com.ionic.sdk.agent.AgentSdk; import com.ionic.sdk.agent.data.MetadataMap; import com.ionic.sdk.agent.key.KeyAttributesMap; import com.ionic.sdk.agent.request.createkey.CreateKeysRequest; @@ -30,7 +25,6 @@ import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase; import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; import com.ionic.sdk.error.IonicException; - import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.EncryptionMaterials; import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; @@ -50,43 +44,38 @@ public class IonicEncryptionMaterialsProvider implements EncryptionMaterialsProv private boolean enabledMetadataCapture = false; - private static KeyAttributesMap defaultAttributes = new KeyAttributesMap(); - private ISAgentPool agentPool = new ISAgentPool(); + private KeyAttributesMap defaultAttributes = new KeyAttributesMap(); + private IonicAgentFactory agentFactory = new IonicAgentFactory(); - private HashMap requestKeyMap = new HashMap(); - private HashMap responseKeyMap = new HashMap(); + private ConcurrentHashMap requestKeyMap = + new ConcurrentHashMap(); + private ConcurrentHashMap responseKeyMap = + new ConcurrentHashMap(); /** * IonicEncryptionMaterialsProvider() default constructor for IonicEncryptionMaterialsProvider */ - public IonicEncryptionMaterialsProvider() { - try { - AgentSdk.initialize(null); - } catch (IonicException e) { - throw new AmazonS3Exception(e.getLocalizedMessage(), e); - } - } + public IonicEncryptionMaterialsProvider() {} /** * IonicEncryptionMaterialsProvider() constructor for IonicEncryptionMaterialsProvider that - * takes a persistor and set it on it's agentPool + * takes a persistor and set it on it's agentFactory * - * @param persistor a {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase} object. + * @param persistor a {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase} + * object. */ public IonicEncryptionMaterialsProvider(DeviceProfilePersistorBase persistor) { try { - AgentSdk.initialize(null); + setPersistor(persistor); } catch (IonicException e) { throw new AmazonS3Exception(e.getLocalizedMessage(), e); } - agentPool.setPersistor(persistor); } /** - * standard() Create new instance of IonicEncryptionMaterialsProvider - * with the default PlainText Persistor. - * Note: PlainText persistors should be used for development only. - * See Ionic devportal for more on profile persistor types + * standard() Create new instance of IonicEncryptionMaterialsProvider with the default PlainText + * Persistor. Note: PlainText persistors should be used for development only. See Ionic + * devportal for more on profile persistor types * https://dev.in.ionicsecurity.com/devportal/develop/platform/concepts/profile-persistor * * @return a {@link com.ionicsecurity.ipcs.awss3.IonicEncryptionMaterialsProvider} object. @@ -96,13 +85,14 @@ static public IonicEncryptionMaterialsProvider standard() { DeviceProfilePersistorPlainText ptPersistor = new DeviceProfilePersistorPlainText(); try { - String sProfilePath = Paths.get(HOME + "/.ionicsecurity/profiles.pt").toFile().getCanonicalPath(); + String sProfilePath = + Paths.get(HOME + "/.ionicsecurity/profiles.pt").toFile().getCanonicalPath(); ptPersistor.setFilePath(sProfilePath); } catch (InvalidPathException e) { - System.err.println("Error: Invalid Path" ); + System.err.println("Error: Invalid Path"); throw new AmazonS3Exception(e.getLocalizedMessage(), e); } catch (IOException e) { - System.err.println("Error: IO Error" ); + System.err.println("Error: IO Error"); throw new AmazonS3Exception(e.getLocalizedMessage(), e); } catch (IonicException e) { throw new AmazonS3Exception(e.getLocalizedMessage(), e); @@ -120,8 +110,8 @@ static public IonicEncryptionMaterialsProvider standard() { public void refresh() {} /** - * getEncryptionMaterials() produces EncryptionMaterials by - * creating a new encryption key via IDC + * getEncryptionMaterials() produces EncryptionMaterials by creating a new encryption key via + * IDC * * @return a {@link com.amazonaws.services.s3.model.EncryptionMaterials} object. */ @@ -136,10 +126,9 @@ public EncryptionMaterials getEncryptionMaterials() { /** * {@inheritDoc} * - * getEncryptionMaterials() produces EncryptionMaterials by either - * creating a new encryption key via IDC if no Ionic_Key_ID key is - * present in materialsDescription or by attempting to retrieve an - * existing key from IDC with its KeyId + * getEncryptionMaterials() produces EncryptionMaterials by either creating a new encryption key + * via IDC if no Ionic_Key_ID key is present in materialsDescription or by attempting to + * retrieve an existing key from IDC with its KeyId */ @Override public EncryptionMaterials getEncryptionMaterials(Map materialsDescription) { @@ -157,11 +146,12 @@ private EncryptionMaterials getEncryptionMaterialsInternal(Map d } } - private EncryptionMaterials generateEncryptionMaterials(Map desc) throws IonicException { + private EncryptionMaterials generateEncryptionMaterials(Map desc) + throws IonicException { KeyAttributesMap kam = new KeyAttributesMap(); if (desc != null && enabledMetadataCapture) { for (Map.Entry entry : desc.entrySet()) { - if (entry.getValue() != IONICKEYREQUUID) { + if (entry.getKey() != IONICKEYREQUUID) { ArrayList collection = new ArrayList(); collection.add(entry.getValue()); kam.put(entry.getKey(), collection); @@ -178,13 +168,9 @@ private EncryptionMaterials generateEncryptionMaterials(Map desc kam.putAll(reqKey.getAttributesMap()); } } - Agent agent = agentPool.getAgent(); + Agent agent = agentFactory.getAgent(); CreateKeysResponse.Key ionicKey = null; - try { - ionicKey = agent.createKey(kam, reqKey.getMutableAttributesMap()).getFirstKey(); - } finally { - agentPool.returnAgent(agent); - } + ionicKey = agent.createKey(kam, reqKey.getMutableAttributesMap()).getFirstKey(); SecretKey awsKey; awsKey = new SecretKeySpec(ionicKey.getKey(), 0, ionicKey.getKey().length, "AES"); EncryptionMaterials materials = new EncryptionMaterials(awsKey); @@ -193,15 +179,12 @@ private EncryptionMaterials generateEncryptionMaterials(Map desc return materials; } - private EncryptionMaterials retrieveEncryptionMaterials(Map desc) throws IonicException { + private EncryptionMaterials retrieveEncryptionMaterials(Map desc) + throws IonicException { String ionicKeyId = desc.get(KEYIDKEY); - Agent agent = agentPool.getAgent(); + Agent agent = agentFactory.getAgent(); GetKeysResponse.Key ionicKey; - try { - ionicKey = agent.getKey(ionicKeyId).getFirstKey(); - } finally { - agentPool.returnAgent(agent); - } + ionicKey = agent.getKey(ionicKeyId).getFirstKey(); storeResponseKey(ionicKey); SecretKey awsKey = new SecretKeySpec(ionicKey.getKey(), 0, ionicKey.getKey().length, "AES"); EncryptionMaterials materials = new EncryptionMaterials(awsKey); @@ -218,11 +201,9 @@ public boolean isEnabledMetadataCapture() { } /** - * setEnabledMetadataCapture() sets enabledMetadataCapture, - * while true S3 requests to store objects with client side - * Encryption will have their userMetadata parsed and passed - * as ionic attributes when content encryption keys are - * generated + * setEnabledMetadataCapture() sets enabledMetadataCapture, while true S3 requests to store + * objects with client side Encryption will have their userMetadata parsed and passed as ionic + * attributes when content encryption keys are generated * * @param enabledMetadataCapture a boolean. */ @@ -230,52 +211,51 @@ public void setEnabledMetadataCapture(boolean enabledMetadataCapture) { this.enabledMetadataCapture = enabledMetadataCapture; } - /** - * setDefaultAttributes() sets the default Attributes to - * be applied to all Agent.keyCreate() requests - * - * @param defaultAttributes a {@link com.ionic.sdk.agent.key.KeyAttributesMap} object. - */ - public static void setDefaultAttributes(KeyAttributesMap defaultAttributes) { - IonicEncryptionMaterialsProvider.defaultAttributes = defaultAttributes; + /** + * setDefaultAttributes() sets the default Attributes to be applied to all Agent.keyCreate() + * requests + * + * @param defaultAttributes a {@link com.ionic.sdk.agent.key.KeyAttributesMap} object. + */ + public void setDefaultAttributes(KeyAttributesMap defaultAttributes) { + this.defaultAttributes = new KeyAttributesMap(defaultAttributes); } - /** - * getDefaultAttributes() gets defaultAttributes - * - * @return a {@link com.ionic.sdk.agent.key.KeyAttributesMap} object. - */ - public static KeyAttributesMap getDefaultAttributes() { - return defaultAttributes; + /** + * getDefaultAttributes() gets defaultAttributes + * + * @return a {@link com.ionic.sdk.agent.key.KeyAttributesMap} object. + */ + public KeyAttributesMap getDefaultAttributes() { + return new KeyAttributesMap(defaultAttributes); } - /** - * setIonicMetadataMap() sets the MetadataMap for IDC interactions - * - * @param map a {@link com.ionic.sdk.agent.data.MetadataMap} object. - */ - public static void setIonicMetadataMap(MetadataMap map) { - ISAgentPool.setMetadataMap(map); + /** + * setIonicMetadataMap() sets the MetadataMap for IDC interactions + * + * @param map a {@link com.ionic.sdk.agent.data.MetadataMap} object. + */ + public void setIonicMetadataMap(MetadataMap map) { + agentFactory.setMetadataMap(map); } - /** - * getIonicMetadataMap() gets the MetadataMap used for IDC interactions - * - * @return a {@link com.ionic.sdk.agent.data.MetadataMap} object. - */ - public static MetadataMap getIonicMetadataMap() { - return ISAgentPool.getMetadataMap(); + /** + * getIonicMetadataMap() gets the MetadataMap used for IDC interactions + * + * @return a {@link com.ionic.sdk.agent.data.MetadataMap} object. + */ + public MetadataMap getIonicMetadataMap() { + return agentFactory.getMetadataMap(); } - /** - * setPersistor() sets the Persistor with which to - * create Agents in the AgentPool - * - * @param persistor a {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase} object. - */ - public void setPersistor(DeviceProfilePersistorBase persistor) - { - agentPool.setPersistor(persistor); + /** + * setPersistor() sets the Persistor with which to create Agents in the AgentPool + * + * @param persistor a {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase} + * object. + */ + public void setPersistor(DeviceProfilePersistorBase persistor) throws IonicException { + agentFactory.setActiveProfile(persistor); } protected String storeRequestKey(CreateKeysRequest.Key key) { @@ -289,7 +269,6 @@ private CreateKeysRequest.Key retrieveRequestKey(String uuid) { } private void storeResponseKey(GetKeysResponse.Key key) { - {} this.responseKeyMap.put(key.getId(), key); } diff --git a/src/main/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClient.java b/src/main/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClient.java similarity index 63% rename from src/main/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClient.java rename to src/main/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClient.java index e61de1e..d3b55a7 100644 --- a/src/main/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClient.java +++ b/src/main/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClient.java @@ -1,20 +1,18 @@ /* - * (c) 2017-2018 Ionic Security Inc. - * By using this code, I agree to the LICENSE included, as well as the - * Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy (https://www.ionic.com/privacy-notice/). + * (c) 2017-2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as + * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). */ -package com.ionicsecurity.ipcs.awss3; +package com.ionic.cloudstorage.awss3; import java.io.File; import java.io.InputStream; import java.io.ByteArrayInputStream; import java.util.Map; import java.util.HashMap; - import com.ionic.sdk.agent.request.createkey.CreateKeysRequest; import com.ionic.sdk.agent.request.getkey.GetKeysResponse; - import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.metrics.RequestMetricCollector; @@ -40,29 +38,24 @@ * A Ionic backed subclass of {@link com.amazonaws.services.s3.AmazonS3EncryptionClient}. */ public class IonicS3EncryptionClient extends AmazonS3EncryptionClient - implements AmazonS3Encryption { + implements AmazonS3Encryption { private IonicEncryptionMaterialsProvider iemp; IonicS3EncryptionClient(IonicS3EncryptionClientParams params) { - this(params.getClientParams().getCredentialsProvider(), - params.getEncryptionMaterials(), - params.getClientParams().getClientConfiguration(), - params.getCryptoConfiguration(), - params.getClientParams().getRequestMetricCollector()); + this(params.getClientParams().getCredentialsProvider(), params.getEncryptionMaterials(), + params.getClientParams().getClientConfiguration(), params.getCryptoConfiguration(), + params.getClientParams().getRequestMetricCollector()); } @SuppressWarnings("deprecation") - private IonicS3EncryptionClient( - AWSCredentialsProvider credentialsProvider, - EncryptionMaterialsProvider kekMaterialsProvider, - ClientConfiguration clientConfig, - CryptoConfiguration cryptoConfig, - RequestMetricCollector requestMetricCollector) { + private IonicS3EncryptionClient(AWSCredentialsProvider credentialsProvider, + EncryptionMaterialsProvider kekMaterialsProvider, ClientConfiguration clientConfig, + CryptoConfiguration cryptoConfig, RequestMetricCollector requestMetricCollector) { super(null, // KMS client - credentialsProvider, kekMaterialsProvider, clientConfig, - cryptoConfig, requestMetricCollector); - this.iemp = (IonicEncryptionMaterialsProvider)kekMaterialsProvider; + credentialsProvider, kekMaterialsProvider, clientConfig, cryptoConfig, + requestMetricCollector); + this.iemp = (IonicEncryptionMaterialsProvider) kekMaterialsProvider; } /** @@ -74,54 +67,61 @@ public PutObjectResult putObject(PutObjectRequest req) { } /** - * A version of {@link #putObject(String, String, File)} - * that takes a {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} - * as an argument for setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} - * and mutableAttributes on the Ionic Key associated with the object. + * A version of {@link #putObject(String, String, File)} that takes a + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} as an argument for + * setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} and mutableAttributes on + * the Ionic Key associated with the object. * * @param bucketName The Bucket to store the Object in. * @param key The key to store the Object under. * @param file The File to stored. * @param ionicKey The CreateKeysRequest.Key containing attributes for associated Ionic Key. - * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the information returned by Amazon S3 for the newly created object. + * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the + * information returned by Amazon S3 for the newly created object. * @see #putObject(PutObjectRequest, CreateKeysRequest.Key) */ - public PutObjectResult putObject(String bucketName, String key, File file, CreateKeysRequest.Key ionicKey) { - return putObject(new PutObjectRequest(bucketName, key, file).withMetadata(new ObjectMetadata()), ionicKey); + public PutObjectResult putObject(String bucketName, String key, File file, + CreateKeysRequest.Key ionicKey) { + return putObject( + new PutObjectRequest(bucketName, key, file).withMetadata(new ObjectMetadata()), + ionicKey); } /** - * A version of {@link #putObject(String, String, InputStream, ObjectMetadata)} - * that takes a {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} - * as an argument for setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} - * and mutableAttributes on the Ionic Key associated with the object. + * A version of {@link #putObject(String, String, InputStream, ObjectMetadata)} that takes a + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} as an argument for + * setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} and mutableAttributes on + * the Ionic Key associated with the object. * * @param bucketName The Bucket to store the Object in. * @param key The key to store the Object under. * @param input InputStream with the Object contents. * @param metadata The ObjectMetadata associated with the Object. * @param ionicKey The CreateKeysRequest.Key containing attributes for associated Ionic Key. - * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the information returned by Amazon S3 for the newly created object. + * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the + * information returned by Amazon S3 for the newly created object. * @see #putObject(PutObjectRequest, CreateKeysRequest.Key) */ - public PutObjectResult putObject(String bucketName, String key, - InputStream input, ObjectMetadata metadata, CreateKeysRequest.Key ionicKey) { + public PutObjectResult putObject(String bucketName, String key, InputStream input, + ObjectMetadata metadata, CreateKeysRequest.Key ionicKey) { return putObject(new PutObjectRequest(bucketName, key, input, metadata), ionicKey); } /** - * A version of {@link #putObject(String, String, String)} - * that takes a {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} - * as an argument for setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} - * and mutableAttributes on the Ionic Key associated with the object. + * A version of {@link #putObject(String, String, String)} that takes a + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} as an argument for + * setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} and mutableAttributes on + * the Ionic Key associated with the object. * * @param bucketName The Bucket to store the Object in. * @param key The key to store the Object under. * @param content A string to store as Object content. - * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the information returned by Amazon S3 for the newly created object. + * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the + * information returned by Amazon S3 for the newly created object. * @see #putObject(PutObjectRequest, CreateKeysRequest.Key) */ - public PutObjectResult putObject(String bucketName, String key, String content, CreateKeysRequest.Key ionicKey) { + public PutObjectResult putObject(String bucketName, String key, String content, + CreateKeysRequest.Key ionicKey) { rejectNull(bucketName, "Bucket name must be provided"); rejectNull(key, "Object key must be provided"); rejectNull(content, "String content must be provided"); @@ -137,31 +137,40 @@ public PutObjectResult putObject(String bucketName, String key, String content, } /** - * A version of {@link #putObject(PutObjectRequest)} - * that takes a {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} - * as an argument for setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} - * and mutableAttributes on the Ionic Key associated with the object. + * A version of {@link #putObject(PutObjectRequest)} that takes a + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} as an argument for + * setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} and mutableAttributes on + * the Ionic Key associated with the object. + * + *

+ * Example of creating a CreateKeysRequest.Key + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key}. + * + * 

+     * {
+     *     @code
+     *     KeyAttributesMap attributes = new KeyAttributesMap();
+     *     KeyAttributesMap mutableAttributes = new KeyAttributesMap();
+     *     attributes.put("Attribute_Key1", Arrays.asList("Val1", "Val2", "Val3"));
+     *     mutableAttributes.put("Mutable_Attribute_Key1", Arrays.asList("Val1", "Val2", "Val3"));
+     *     CreateKeysRequest.Key reqKey =
+     *             new CreateKeysRequest.Key("", 1, attributes, mutableAttributes);
+     * }
+     *
* - *

Example of creating a CreateKeysRequest.Key {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key}. - * 

 {@code
-     * KeyAttributesMap attributes = new KeyAttributesMap();
-     * KeyAttributesMap mutableAttributes = new KeyAttributesMap();
-     * attributes.put("Attribute_Key1", Arrays.asList("Val1","Val2","Val3"));
-     * mutableAttributes.put("Mutable_Attribute_Key1", Arrays.asList("Val1","Val2","Val3"));
-     * CreateKeysRequest.Key reqKey = new CreateKeysRequest.Key("", 1, attributes, mutableAttributes);
-     * }
* @param req The PutObjectRequest object that specifies all the parameters of this operation. * @param key The CreateKeysRequest.Key containing attributes for associated Ionic Key. - * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the information returned by Amazon S3 for the newly created object. + * @return A {@link com.amazonaws.services.s3.model.PutObjectResult} object containing the + * information returned by Amazon S3 for the newly created object. */ public PutObjectResult putObject(PutObjectRequest req, CreateKeysRequest.Key key) { EncryptedPutObjectRequest cryptoReq; if (req instanceof EncryptedPutObjectRequest) { - cryptoReq = ((EncryptedPutObjectRequest)req); + cryptoReq = ((EncryptedPutObjectRequest) req); } else { if (req.getInputStream() != null) { - cryptoReq = new EncryptedPutObjectRequest(req.getBucketName(), - req.getKey(), req.getInputStream(), req.getMetadata()); + cryptoReq = new EncryptedPutObjectRequest(req.getBucketName(), req.getKey(), + req.getInputStream(), req.getMetadata()); } else if (req.getFile() != null) { cryptoReq = new EncryptedPutObjectRequest(req.getBucketName(), req.getKey(), req.getFile()); @@ -175,21 +184,24 @@ public PutObjectResult putObject(PutObjectRequest req, CreateKeysRequest.Key key if (iemp.isEnabledMetadataCapture()) { ObjectMetadata objMetadata = req.getMetadata(); if (objMetadata != null) { - Map userMetadata = objMetadata.getUserMetadata(); + Map userMetadata = objMetadata.getUserMetadata(); if (userMetadata != null) { materialsDescription.putAll(userMetadata); } } } - materialsDescription.put(IonicEncryptionMaterialsProvider.IONICKEYREQUUID, iemp.storeRequestKey(key)); + materialsDescription.put(IonicEncryptionMaterialsProvider.IONICKEYREQUUID, + iemp.storeRequestKey(key)); cryptoReq.setMaterialsDescription(materialsDescription); return super.putObject(cryptoReq); } - private GetKeysResponse.Key keyFromMetadataInternal(ObjectMetadata meta) - { + private GetKeysResponse.Key keyFromMetadataInternal(ObjectMetadata meta) { String matdesc = meta.getUserMetaDataOf("x-amz-matdesc"); + if (matdesc == null) { + return new GetKeysResponse.Key(); + } matdesc = matdesc.substring(1, matdesc.length()); String[] descriptions = matdesc.split(","); String keyId = null; @@ -203,8 +215,10 @@ private GetKeysResponse.Key keyFromMetadataInternal(ObjectMetadata meta) } /** - * A container class that holds a pairing of {@link com.ionic.sdk.agent.request.createkey.CreateKeysResponse.Key} - * and {@link com.amazonaws.services.s3.model.S3Object} returned by {@link #getObjectAndKey()} methods. + * A container class that holds a pairing of + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysResponse.Key} and + * {@link com.amazonaws.services.s3.model.S3Object} returned by {@link #getObjectAndKey()} + * methods. */ public class IonicKeyS3ObjectPair { private GetKeysResponse.Key key; @@ -217,6 +231,7 @@ private IonicKeyS3ObjectPair(GetKeysResponse.Key key, S3Object object) { /** * Returns a GetKeysResponse.Key. + * * @return a {@link com.ionic.sdk.agent.request.createkey.CreateKeysResponse.Key} */ public GetKeysResponse.Key getKey() { @@ -225,6 +240,7 @@ public GetKeysResponse.Key getKey() { /** * Returns a S3Object. + * * @return a {@link com.amazonaws.services.s3.model.S3Object} */ public S3Object getS3Object() { @@ -244,9 +260,10 @@ public S3Object getObject(GetObjectRequest req) { } /** - * A version of {@link #getObject(String, String)} that returns a IonicKeyS3ObjectPair containing - * the requested S3Object and the GetKeysResponse.Key for the underlying Ionic + * A version of {@link #getObject(String, String)} that returns a IonicKeyS3ObjectPair + * containing the requested S3Object and the GetKeysResponse.Key for the underlying Ionic * {@link com.ionic.sdk.agent.Agent#getKey(String)} request. + * * @param bucketName The name of the bucket containing the desired object. * @param key The key under which the desired object is stored. * @return a {@link IonicS3EncryptionClient.IonicKeyS3ObjectPair} @@ -256,9 +273,10 @@ public IonicKeyS3ObjectPair getObjectAndKey(String bucketName, String key) { } /** - * A version of {@link #getObject(GetObjectRequest)} that returns a IonicKeyS3ObjectPair containing - * the requested S3Object and the GetKeysResponse.Key for the underlying Ionic + * A version of {@link #getObject(GetObjectRequest)} that returns a IonicKeyS3ObjectPair + * containing the requested S3Object and the GetKeysResponse.Key for the underlying Ionic * {@link com.ionic.sdk.agent.Agent#getKey(String)} request. + * * @param req The request object containing all the options on how to download the object. * @return a {@link IonicS3EncryptionClient.IonicKeyS3ObjectPair} */ @@ -269,8 +287,10 @@ public IonicKeyS3ObjectPair getObjectAndKey(GetObjectRequest req) { } /** - * A container class that holds a pairing of {@link com.ionic.sdk.agent.request.createkey.CreateKeysResponse.Key} - * and {@link com.amazonaws.services.s3.model.ObjectMetadata} returned by {@link #readAllBytesAndKey()} methods. + * A container class that holds a pairing of + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysResponse.Key} and + * {@link com.amazonaws.services.s3.model.ObjectMetadata} returned by + * {@link #readAllBytesAndKey()} methods. */ public class IonicKeyObjectMetadataPair { private GetKeysResponse.Key key; @@ -283,6 +303,7 @@ private IonicKeyObjectMetadataPair(GetKeysResponse.Key key, ObjectMetadata meta) /** * Returns a GetKeysResponse.Key. + * * @return a {@link com.ionic.sdk.agent.request.createkey.CreateKeysResponse.Key} */ public GetKeysResponse.Key getKey() { @@ -291,6 +312,7 @@ public GetKeysResponse.Key getKey() { /** * Returns a ObjectMetadata. + * * @return a {@link com.amazonaws.services.s3.model.ObjectMetadata} */ public ObjectMetadata getObjectMetadata() { @@ -309,11 +331,14 @@ public ObjectMetadata getObject(GetObjectRequest req, File dest) { } /** - * A version of {@link #getObject(GetObjectRequest, File)} that returns a IonicKeyObjectMetadataPair containing - * the requested ObjectMetadata and the GetKeysResponse.Key for the underlying Ionic - * {@link com.ionic.sdk.agent.Agent#getKey(String)} request. + * A version of {@link #getObject(GetObjectRequest, File)} that returns a + * IonicKeyObjectMetadataPair containing the requested ObjectMetadata and the + * GetKeysResponse.Key for the underlying Ionic {@link com.ionic.sdk.agent.Agent#getKey(String)} + * request. + * * @param req The request object containing all the options on how to download the object. - * @param dest Indicates the file (which might already exist) where to save the object content being downloading from Amazon S3. + * @param dest Indicates the file (which might already exist) where to save the object content + * being downloading from Amazon S3. * @return a {@link IonicS3EncryptionClient.IonicKeyObjectMetadataPair} */ public IonicKeyObjectMetadataPair getObjectAndKey(GetObjectRequest req, File dest) { @@ -331,49 +356,60 @@ public InitiateMultipartUploadResult initiateMultipartUpload( } /** - * A version of {@link #initiateMultipartUpload(InitiateMultipartUploadRequest)} - * that takes a {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} - * as an argument for setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} - * and mutableAttributes on the Ionic Key associated with the object. + * A version of {@link #initiateMultipartUpload(InitiateMultipartUploadRequest)} that takes a + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key} as an argument for + * setting {@link com.ionic.sdk.agent.key.KeyAttributesMap Attributes} and mutableAttributes on + * the Ionic Key associated with the object. * - *

Example of creating a CreateKeysRequest.Key {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key}. - * 

 {@code
-     * KeyAttributesMap attributes = new KeyAttributesMap();
-     * KeyAttributesMap mutableAttributes = new KeyAttributesMap();
-     * attributes.put("Attribute_Key1", Arrays.asList("Val1","Val2","Val3"));
-     * mutableAttributes.put("Mutable_Attribute_Key1", Arrays.asList("Val1","Val2","Val3"));
-     * CreateKeysRequest.Key reqKey = new CreateKeysRequest.Key("", 1, attributes, mutableAttributes);
-     * }
- * @param req The InitiateMultipartUploadRequest object that specifies all the parameters of this operation. + *

+ * Example of creating a CreateKeysRequest.Key + * {@link com.ionic.sdk.agent.request.createkey.CreateKeysRequest.Key}. + * + * 

+     * {
+     *     @code
+     *     KeyAttributesMap attributes = new KeyAttributesMap();
+     *     KeyAttributesMap mutableAttributes = new KeyAttributesMap();
+     *     attributes.put("Attribute_Key1", Arrays.asList("Val1", "Val2", "Val3"));
+     *     mutableAttributes.put("Mutable_Attribute_Key1", Arrays.asList("Val1", "Val2", "Val3"));
+     *     CreateKeysRequest.Key reqKey =
+     *             new CreateKeysRequest.Key("", 1, attributes, mutableAttributes);
+     * }
+     *
+ * + * @param req The InitiateMultipartUploadRequest object that specifies all the parameters of + * this operation. * @param key The CreateKeysRequest.Key containing attributes for associated Ionic Key. * @return An InitiateMultipartUploadResult from Amazon S3. */ - public InitiateMultipartUploadResult initiateMultipartUpload( - InitiateMultipartUploadRequest req, CreateKeysRequest.Key key) { + public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest req, + CreateKeysRequest.Key key) { EncryptedInitiateMultipartUploadRequest cryptoReq; if (req instanceof EncryptedInitiateMultipartUploadRequest) { - cryptoReq = ((EncryptedInitiateMultipartUploadRequest)req); + cryptoReq = ((EncryptedInitiateMultipartUploadRequest) req); } else { - cryptoReq = new EncryptedInitiateMultipartUploadRequest( - req.getBucketName(), req.getKey(), req.getObjectMetadata()); + cryptoReq = new EncryptedInitiateMultipartUploadRequest(req.getBucketName(), + req.getKey(), req.getObjectMetadata()); } HashMap materialsDescription = new HashMap(); if (iemp.isEnabledMetadataCapture()) { ObjectMetadata objMetadata = req.getObjectMetadata(); if (objMetadata != null) { - Map userMetadata = objMetadata.getUserMetadata(); + Map userMetadata = objMetadata.getUserMetadata(); if (userMetadata != null) { materialsDescription.putAll(userMetadata); } } } - materialsDescription.put(IonicEncryptionMaterialsProvider.IONICKEYREQUUID, iemp.storeRequestKey(key)); + materialsDescription.put(IonicEncryptionMaterialsProvider.IONICKEYREQUUID, + iemp.storeRequestKey(key)); cryptoReq.setMaterialsDescription(materialsDescription); return super.initiateMultipartUpload(cryptoReq); } private void rejectNull(Object parameterValue, String errorMessage) { - if (parameterValue == null) throw new IllegalArgumentException(errorMessage); + if (parameterValue == null) + throw new IllegalArgumentException(errorMessage); } } diff --git a/src/main/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientBuilder.java b/src/main/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientBuilder.java similarity index 64% rename from src/main/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientBuilder.java rename to src/main/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientBuilder.java index 5e00e85..92845ea 100644 --- a/src/main/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientBuilder.java +++ b/src/main/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientBuilder.java @@ -1,12 +1,10 @@ /* - * (c) 2017-2018 Ionic Security Inc. - * By using this code, I agree to the LICENSE included, as well as the - * Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy (https://www.ionic.com/privacy-notice/). + * (c) 2017-2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as + * the Terms & Conditions (https://dev.ionic.com/use) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). */ -package com.ionicsecurity.ipcs.awss3; - -import com.ionicsecurity.ipcs.awss3.IonicEncryptionMaterialsProvider; +package com.ionic.cloudstorage.awss3; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.client.AwsSyncClientParams; @@ -18,10 +16,12 @@ /** * IonicS3EncryptionClientBuilder class. * - * A factory class for building instances of {@link com.ionicsecurity.ipcs.awss3.IonicS3EncryptionClient}. - * Mirrors {@link com.amazonaws.services.s3.AmazonS3EncryptionClientBuilder} + * A factory class for building instances of + * {@link com.ionicsecurity.ipcs.awss3.IonicS3EncryptionClient}. Mirrors + * {@link com.amazonaws.services.s3.AmazonS3EncryptionClientBuilder} */ -public class IonicS3EncryptionClientBuilder extends AmazonS3Builder { +public class IonicS3EncryptionClientBuilder + extends AmazonS3Builder { private EncryptionMaterialsProvider encryptionMaterials; private CryptoConfiguration cryptoConfig = new CryptoConfiguration(); @@ -29,19 +29,23 @@ public class IonicS3EncryptionClientBuilder extends AmazonS3Builder agentStack = new Stack(); - private DeviceProfilePersistorBase persistor; - - /** - * getAgent() acquires an Agent from the AgentPool - * - * @return a {@link com.ionic.sdk.agent.Agent} object. - * @throws com.ionic.sdk.error.IonicException if the - * {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase Persistor} - * is unset or does not contain an active profile. - */ - public Agent getAgent() throws IonicException - { - Agent agent; - - poolLock.lock(); - if (agentStack.isEmpty() == true) - { - poolLock.unlock(); - agent = new Agent(); - if (persistor == null) - { - throw new IonicException(AgentErrorModuleConstants.ISAGENT_NO_DEVICE_PROFILE); - } - agent.initialize(persistor); - // Protect against npe when no active profile set - if (agent.hasActiveProfile() == false) { - throw new IonicException(AgentErrorModuleConstants.ISAGENT_NO_DEVICE_PROFILE); - } - agent.setMetadata(metadataMap); - return agent; - } - agent = agentStack.pop(); - poolLock.unlock(); - return agent; - } - - /** - * returnAgent() returns an Agent to the AgentPool - * It is no longer to safe to reference the agent afterwards - * - * @param a a {@link com.ionic.sdk.agent.Agent} object. - */ - public void returnAgent(Agent a) - { - poolLock.lock(); - agentStack.push(a); - poolLock.unlock(); - } - - /** - * flush() empties the AgentPool - */ - public void flush() - { - poolLock.lock(); - agentStack.removeAllElements(); - poolLock.unlock(); - } - - /** - * setPersistor() sets the persistor to use when making - * new agents and empties the AgentPool - * - * @param persistor a {@link com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorBase} object. - */ - public void setPersistor(DeviceProfilePersistorBase persistor) - { - this.persistor = persistor; - this.flush(); - } - - /** - * setMetadataMap() sets the MetadataMap to be - * used when generating new Agents - * - * @param map a {@link com.ionic.sdk.agent.data.MetadataMap} object. - */ - static void setMetadataMap(MetadataMap map) - { - metadataMap = map; - } - - /** - * getIonicMetadataMap() gets Agent.MetaDataMap - * - * @return a {@link com.ionic.sdk.agent.data.MetadataMap} object. - */ - static MetadataMap getMetadataMap() - { - return metadataMap; - } - -} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderFunctionalTest.java b/src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderFunctionalTest.java new file mode 100644 index 0000000..2d77e0e --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderFunctionalTest.java @@ -0,0 +1,74 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import static org.junit.Assert.*; + +import com.amazonaws.services.s3.model.EncryptionMaterials; +import com.ionic.sdk.agent.Agent; +import com.ionic.sdk.agent.request.getkey.GetKeysResponse; +import com.ionic.sdk.agent.request.createkey.CreateKeysResponse; +import com.ionic.sdk.error.IonicException; +import java.util.HashMap; +import javax.crypto.spec.SecretKeySpec; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class IonicEncryptionMaterialsProviderFunctionalTest { + + private static IonicEncryptionMaterialsProvider iemp = null; + private static Agent agent = null; + + @BeforeClass + public static void setup() { + try { + iemp = TestUtils.getIEMP(); + agent = TestUtils.getAgent(); + } catch (IonicException e) { + // Catch any IonicExceptions thrown during setup and null related objects so + // that dependent tests are each skipped during the preconditions check. + iemp = null; + agent = null; + } + } + + @Before + public void preconditions() { + Assume.assumeNotNull(iemp); + Assume.assumeNotNull(agent); + } + + @Test + public void getEncryptionMaterials() throws IonicException { + EncryptionMaterials encMat = iemp.getEncryptionMaterials(null); + + String ionicKeyId = encMat.getMaterialsDescription().get(IonicEncryptionMaterialsProvider.KEYIDKEY); + assertNotNull("Generated EncryptionMaterials did not contain Ionic Key ID", ionicKeyId); + + GetKeysResponse.Key IonicKey = null; + IonicKey = agent.getKey(ionicKeyId).getFirstKey(); + } + + @Test + public void getEncryptionMaterialsWithDesc() throws IonicException { + HashMap desc = new HashMap(); + + CreateKeysResponse.Key IonicKey = agent.createKey().getFirstKey(); + desc.put(IonicEncryptionMaterialsProvider.KEYIDKEY, IonicKey.getId()); + + EncryptionMaterials encMat = iemp.getEncryptionMaterials(desc); + + SecretKeySpec awsKey = new SecretKeySpec(IonicKey.getKey(), 0, IonicKey.getKey().length, "AES"); + assertTrue("SecretKey returned from getEncryptionMaterials() did not match SecretKey derived from original Ionic Key.", + encMat.getSymmetricKey().equals(awsKey)); + } + + +} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderUnitTest.java b/src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderUnitTest.java new file mode 100644 index 0000000..edcb1a8 --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/IonicEncryptionMaterialsProviderUnitTest.java @@ -0,0 +1,84 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import static org.junit.Assert.*; + +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.ionic.sdk.agent.data.MetadataMap; +import com.ionic.sdk.agent.key.KeyAttributesMap; +import com.ionic.sdk.device.profile.DeviceProfile; +import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; +import com.ionic.sdk.error.IonicException; +import java.util.ArrayList; +import org.junit.Rule; +import org.junit.rules.ExpectedException; +import org.junit.Test; + + +public class IonicEncryptionMaterialsProviderUnitTest { + + @Test + public void defaultIEMPConstructor() { + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); + } + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void persistorIEMPConstructor() throws AmazonS3Exception { + DeviceProfilePersistorPlainText ptPersistor = new DeviceProfilePersistorPlainText(); + thrown.expect(AmazonS3Exception.class); + thrown.expectMessage("40022 - No active device profile is set (Service: null; Status" + + " Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null)"); + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(ptPersistor); + } + + @Test + public void setGetMetaDataCapture() { + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); + assertFalse("MetadataCapture was not false by default.", iemp.isEnabledMetadataCapture()); + iemp.setEnabledMetadataCapture(true); + assertTrue("MetadataCapture was not true after iemp.setEnabledMetadataCapture(true)", iemp.isEnabledMetadataCapture()); + } + + @Test + public void setGetDefaultMetadata() { + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); + assertTrue("DefaultAttributes were not empty by default.", iemp.getDefaultAttributes().isEmpty()); + + KeyAttributesMap kam = new KeyAttributesMap(); + ArrayList collection = new ArrayList(); + collection.add("confidential"); + collection.add("secured"); + kam.put("privacy", collection); + + iemp.setDefaultAttributes(kam); + assertEquals("getDefaultAttributes() did not equal map set with setDefaultAttributes()", iemp.getDefaultAttributes(), kam); + } + + @Test + public void setGetAgentMetadata() { + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); + assertTrue("IonicMetadataMap was not empty by default.", iemp.getIonicMetadataMap().isEmpty()); + + MetadataMap metaMap = new MetadataMap(); + metaMap.set("ionic-application-name", "Unit_Test"); + metaMap.set("ionic-application-version", "0.0.0"); + + iemp.setIonicMetadataMap(metaMap); + assertEquals("getIonicMetadataMap() did not equal map set with setIonicMetadataMap()", iemp.getIonicMetadataMap(), metaMap); + } + + @Test + public void refresh() { + IonicEncryptionMaterialsProvider iemp = new IonicEncryptionMaterialsProvider(); + iemp.refresh(); + } + +} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientBuilderUnitTest.java b/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientBuilderUnitTest.java new file mode 100644 index 0000000..1ed1afc --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientBuilderUnitTest.java @@ -0,0 +1,94 @@ +/* + * (c) 2017-2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import static org.junit.Assert.*; + +import java.util.Map; +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.model.EncryptionMaterials; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + +public class IonicS3EncryptionClientBuilderUnitTest { + + static Boolean awsCredsAvailable; + + @BeforeClass + public static void setup() { + awsCredsAvailable = TestUtils.awsCredsAvailable(); + } + + @Test + public void builderWithNonIonicMaterialsProvider() { + + class NonIonicEncryptionMaterialsProvider implements EncryptionMaterialsProvider { + @Override + public EncryptionMaterials getEncryptionMaterials( + Map materialsDescription) { + return null; + } + + @Override + public EncryptionMaterials getEncryptionMaterials() { + return null; + } + + @Override + public void refresh() {} + } + + IonicS3EncryptionClientBuilder ionicBuilder = new IonicS3EncryptionClientBuilder(); + + NonIonicEncryptionMaterialsProvider materialsProvider = new NonIonicEncryptionMaterialsProvider(); + ionicBuilder.setEncryptionMaterials(materialsProvider); + + try { + AmazonS3Encryption ionicEncryptionClient = ionicBuilder.build(); + fail("Non-Ionic EncryptionMaterialsProvider was successfully used to instantiate an IonicS3EncryptionClient"); + } catch (IllegalArgumentException e) { + assertTrue(e instanceof java.lang.IllegalArgumentException); + } catch (Exception e) { + fail("Unexpected Exception building IonicS3EncryptionClient with Non-Ionic EncryptionMaterialsProvider "); + } + } + + @Test + public void builderWithoutProvider() { + try { + AmazonS3Encryption ionicEncryptionClient = IonicS3EncryptionClientBuilder.standard().build(); + fail("Expected an Illegal Argument Exception to be thrown"); + } catch (IllegalArgumentException e) { + assertTrue(e instanceof java.lang.IllegalArgumentException); + } catch (Exception e) { + fail("Unexpected Exception building IonicS3EncryptionClient without EncryptionMaterialsProvider "); + } + } + + @Test + public void builderWithProvider() { + Assume.assumeTrue(awsCredsAvailable); + AmazonS3Encryption ionicEncryptionClient = null; + try { + ionicEncryptionClient = IonicS3EncryptionClientBuilder.standard() + .withEncryptionMaterials(new IonicEncryptionMaterialsProvider()).build(); + } catch (Exception e) { + fail("Unexpected Exception building IonicS3EncryptionClient with IonicMaterialsProvider"); + } + assertTrue(ionicEncryptionClient instanceof IonicS3EncryptionClient); + } + + @Test + public void defaultClient() { + Assume.assumeTrue(awsCredsAvailable); + AmazonS3Encryption ionicEncryptionClient = IonicS3EncryptionClientBuilder.defaultClient(); + assertTrue(ionicEncryptionClient instanceof IonicS3EncryptionClient); + } + +} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientDeniedTest.java b/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientDeniedTest.java new file mode 100644 index 0000000..22b9944 --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientDeniedTest.java @@ -0,0 +1,75 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import static org.junit.Assert.*; + +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.ionic.sdk.agent.Agent; +import com.ionic.sdk.error.IonicException; +import java.io.IOException; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.ExpectedException; +import org.junit.Test; + +public class IonicS3EncryptionClientDeniedTest { + + private static IonicEncryptionMaterialsProvider iemp = null; + private static IonicS3EncryptionClient ionicS3Client = null; + private static String testBucket = null; + private static String testString = null; + + @BeforeClass + public static void setup() { + if (TestUtils.awsCredsAvailable()) { + try { + iemp = TestUtils.getIEMP(); + ionicS3Client = (IonicS3EncryptionClient)IonicS3EncryptionClientBuilder.standard() + .withEncryptionMaterials(iemp).build(); + } catch (IonicException e) { + // Catch any IonicExceptions thrown during setup and null related objects so + // that dependent tests are each skipped during the preconditions check. + iemp = null; + ionicS3Client = null; + } + } + testBucket = TestUtils.getTestBucket(); + testString = TestUtils.getTestPayload(); + } + + @Before + public void preconditions() { + Assume.assumeTrue(TestUtils.isProfilePolicyDenied()); + Assume.assumeNotNull(ionicS3Client); + Assume.assumeNotNull(testBucket); + } + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void putAndGetObject() throws IOException, AmazonS3Exception { + + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testPutAndGetObject"; + } + + ionicS3Client.putObject(testBucket, key, testString); + + thrown.expect(AmazonS3Exception.class); + thrown.expectMessage("40024 - Key fetch or creation was denied by the server " + + "(Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null"); + + ionicS3Client.getObject(testBucket, key); + } + + +} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientFunctionalTest.java b/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientFunctionalTest.java new file mode 100644 index 0000000..8243929 --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/IonicS3EncryptionClientFunctionalTest.java @@ -0,0 +1,357 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import static org.junit.Assert.*; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ionic.sdk.agent.Agent; +import com.ionic.sdk.agent.data.MetadataMap; +import com.ionic.sdk.agent.key.KeyAttributesMap; +import com.ionic.sdk.agent.request.createkey.CreateKeysRequest; +import com.ionic.sdk.agent.request.getkey.GetKeysResponse; +import com.ionic.sdk.error.IonicException; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.Writer; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.concurrent.TimeoutException; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class IonicS3EncryptionClientFunctionalTest { + + private static IonicEncryptionMaterialsProvider iemp = null; + private static IonicS3EncryptionClient ionicS3Client = null; + private static Agent agent = null; + private static AmazonS3 s3Client = null; + private static String testBucket = null; + private static String testString = null; + + @BeforeClass + public static void setup() { + if (TestUtils.awsCredsAvailable()) { + try { + iemp = TestUtils.getIEMP(); + ionicS3Client = (IonicS3EncryptionClient)IonicS3EncryptionClientBuilder.standard() + .withEncryptionMaterials(iemp).build(); + agent = TestUtils.getAgent(); + } catch (IonicException e) { + // Catch any IonicExceptions thrown during setup and null related objects so + // that dependent tests are each skipped during the preconditions check. + iemp = null; + ionicS3Client = null; + agent = null; + } + s3Client = AmazonS3ClientBuilder.defaultClient(); + } + testBucket = TestUtils.getTestBucket(); + testString = TestUtils.getTestPayload(); + } + + @Before + public void preconditions() { + Assume.assumeNotNull(ionicS3Client); + Assume.assumeNotNull(testBucket); + Assume.assumeNotNull(agent); + Assume.assumeNotNull(iemp); + } + + @Test + public void putAndGetObject() throws IOException, TimeoutException, InterruptedException { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testPutAndGetObject"; + } + + ionicS3Client.putObject(testBucket, key, testString); + waitUntilObjectExists(testBucket, key); + S3Object obj = ionicS3Client.getObject(testBucket, key); + assertTrue("Decrypted object content does not match original String", + IOUtils.toString(obj.getObjectContent(), "UTF8").equals(testString)); + } + + @Test + public void atRestEncryption() throws IOException, TimeoutException, InterruptedException { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testAtRestEncryption"; + } + + ionicS3Client.putObject(testBucket, key, testString); + waitUntilObjectExists(testBucket, key); + S3Object obj = s3Client.getObject(testBucket, key); + assertFalse("Uloaded object content matches original String", + IOUtils.toString(obj.getObjectContent(), "UTF8").equals(testString)); + } + + @Test + public void metadataCaptureOn() throws IOException, IonicException, TimeoutException, InterruptedException { + String testMetaKey = "TestMetadataKey"; + String testMetaValue = "TestMetadataValue"; + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testMetadataCaptureOn"; + } + + iemp.setEnabledMetadataCapture(true); + + InputStream iStream = IOUtils.toInputStream(testString); + + ObjectMetadata requestMetadata = new ObjectMetadata(); + requestMetadata.addUserMetadata(testMetaKey, testMetaValue); + requestMetadata.setContentLength(testString.length()); + + PutObjectRequest request = new PutObjectRequest(testBucket, key, iStream, requestMetadata); + ionicS3Client.putObject(request); + waitUntilObjectExists(testBucket, key); + ObjectMetadata uploadedMetadata = s3Client.getObjectMetadata(testBucket, key); + String ionicKeyId = extractIonicKeyIDFromMatdesc(uploadedMetadata.getUserMetaDataOf("x-amz-matdesc")); + + GetKeysResponse.Key ionicKey = agent.getKey(ionicKeyId).getFirstKey(); + + assertTrue( "Captured metadata value is not equal to \"" + testMetaValue + '"', + ionicKey.getAttributesMap().get(testMetaKey).get(0).equals(testMetaValue)); + } + + @Test + public void metadataCaptureOff() throws IOException, IonicException, TimeoutException, InterruptedException { + String testMetaKey = "TestMetadataKey"; + String testMetaValue = "TestMetadataValue"; + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testMetadataCaptureOff"; + } + + iemp.setEnabledMetadataCapture(false); + + InputStream iStream = IOUtils.toInputStream(testString); + + ObjectMetadata requestMetadata = new ObjectMetadata(); + requestMetadata.addUserMetadata(testMetaKey, testMetaValue); + requestMetadata.setContentLength(testString.length()); + + PutObjectRequest request = new PutObjectRequest(testBucket, key, iStream, requestMetadata); + ionicS3Client.putObject(request); + waitUntilObjectExists(testBucket, key); + ObjectMetadata uploadedMetadata = s3Client.getObjectMetadata(testBucket, key); + String ionicKeyId = extractIonicKeyIDFromMatdesc(uploadedMetadata.getUserMetaDataOf("x-amz-matdesc")); + + GetKeysResponse.Key ionicKey = agent.getKey(ionicKeyId).getFirstKey(); + + assertTrue( "Metadata key \"" + testMetaKey + "\" is present in IonicKey Attributes", + ionicKey.getAttributesMap().get(testMetaKey) == null); + } + + @Test + public void putObjectWithAttributes() throws IOException, IonicException, TimeoutException, InterruptedException { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testPutObjectWithAttributes"; + } + + iemp.setEnabledMetadataCapture(true); + + KeyAttributesMap attributes = new KeyAttributesMap(); + KeyAttributesMap mutableAttributes = new KeyAttributesMap(); + attributes.put("Attribute", Arrays.asList("Val1", "Val2", "Val3")); + mutableAttributes.put("Mutable-Attribute", Arrays.asList("Val1", "Val2", "Val3")); + CreateKeysRequest.Key ionicRequestKey = new CreateKeysRequest.Key("", 1, attributes, mutableAttributes); + + ionicS3Client.putObject(testBucket, key, testString, ionicRequestKey); + waitUntilObjectExists(testBucket, key); + ObjectMetadata uploadedMetadata = s3Client.getObjectMetadata(testBucket, key); + String ionicKeyId = extractIonicKeyIDFromMatdesc(uploadedMetadata.getUserMetaDataOf("x-amz-matdesc")); + + GetKeysResponse.Key ionicKey = agent.getKey(ionicKeyId).getFirstKey(); + + assertTrue("Response Key Attributes do not match specified Attributes", + ionicKey.getAttributesMap().equals(attributes)); + + assertTrue("Response Key Mutable Attributes do not match specified Mutable Attributes", + ionicKey.getMutableAttributesMap().equals(mutableAttributes)); + } + + @Test + public void putAndGetObjectWithAttributes() throws IOException, TimeoutException, InterruptedException { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testPutAndGetObjectWithAttributes"; + } + + iemp.setEnabledMetadataCapture(true); + + KeyAttributesMap attributes = new KeyAttributesMap(); + KeyAttributesMap mutableAttributes = new KeyAttributesMap(); + attributes.put("Attribute", Arrays.asList("Val1", "Val2", "Val3")); + mutableAttributes.put("Mutable-Attribute", Arrays.asList("Val1", "Val2", "Val3")); + CreateKeysRequest.Key ionicRequestKey = new CreateKeysRequest.Key("", 1, attributes, mutableAttributes); + + ionicS3Client.putObject(testBucket, key, testString, ionicRequestKey); + waitUntilObjectExists(testBucket, key); + IonicS3EncryptionClient.IonicKeyS3ObjectPair pair = ionicS3Client.getObjectAndKey(testBucket, key); + S3Object obj = pair.getS3Object(); + GetKeysResponse.Key ionicKey = pair.getKey(); + + assertTrue("Response Key Attributes do not match specified Attributes", + ionicKey.getAttributesMap().equals(attributes)); + + assertTrue("Response Key Mutable Attributes do not match specified Mutable Attributes", + ionicKey.getMutableAttributesMap().equals(mutableAttributes)); + + assertTrue("Decrypted object content does not match original String", + IOUtils.toString(obj.getObjectContent(), "UTF8").equals(testString)); + } + + @Test + public void unencryptedDownload() throws IOException { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testUnencryptedDownload"; + } + s3Client.putObject(testBucket, key, testString); + + S3Object object = ionicS3Client.getObject(testBucket, key); + String contents = IOUtils.toString(object.getObjectContent()); + assertTrue("Unencrypted download did not match source", testString.equals(contents)); + } + + @Test + public void multipartUpload() throws IOException, AmazonS3Exception { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testMultipartUpload"; + } + + iemp.setEnabledMetadataCapture(true); + + InputStream inStream = null; + long sourceSize = 0; + File sourceFile = TestUtils.getSourceFile(); + File destFile = TestUtils.getDestFile(); + Assume.assumeNotNull(sourceFile); + Assume.assumeNotNull(destFile); + + KeyAttributesMap attributes = new KeyAttributesMap(); + KeyAttributesMap mutableAttributes = new KeyAttributesMap(); + attributes.put("Attribute", Arrays.asList("Val1", "Val2", "Val3")); + mutableAttributes.put("Mutable-Attribute", Arrays.asList("Val1", "Val2", "Val3")); + + CreateKeysRequest.Key reqKey = + new CreateKeysRequest.Key("", 1, attributes, mutableAttributes); + + uploadMultipart(key, sourceFile, reqKey); + inStream = new FileInputStream(sourceFile); + + IonicS3EncryptionClient.IonicKeyS3ObjectPair pair; + pair = ionicS3Client.getObjectAndKey(testBucket, key); + + S3Object s3Object = pair.getS3Object(); + FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), destFile); + + assertTrue("Response Key Attributes do not match specified Attributes", + pair.getKey().getAttributesMap().equals(attributes)); + + assertTrue("Response Key Mutable Attributes do not match specified Mutable Attributes", + pair.getKey().getMutableAttributesMap().equals(mutableAttributes)); + + assertTrue("Downloaded File did not match original File", + FileUtils.contentEquals(sourceFile, destFile)); + } + + public void uploadMultipart(String key, File sourceFile, CreateKeysRequest.Key ionicKey) + throws FileNotFoundException, IOException { + float totalChunks; + long partSizeBytes = 6 * 1024 * 1024; + long contentLength = sourceFile.length(); + ArrayList partETags = new ArrayList(); + + if (contentLength % partSizeBytes != 0) { + totalChunks = contentLength / partSizeBytes + 1; + } else { + totalChunks = contentLength / partSizeBytes; + } + + ObjectMetadata s3ObjectMetadata = new ObjectMetadata(); + + InitiateMultipartUploadRequest req = + new InitiateMultipartUploadRequest(testBucket, key, s3ObjectMetadata); + InitiateMultipartUploadResult res = ionicS3Client.initiateMultipartUpload(req, ionicKey); + + try { + long filePosition = 0; + for (int i = 1; filePosition < contentLength; i++) { + partSizeBytes = Math.min(partSizeBytes, (contentLength - filePosition)); + boolean isLast = i == totalChunks; + + UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(testBucket) + .withKey(key).withUploadId(res.getUploadId()).withPartNumber(i) + .withFileOffset(filePosition).withFile(sourceFile) + .withPartSize(partSizeBytes).withLastPart(isLast); + + partETags.add(ionicS3Client.uploadPart(uploadRequest).getPartETag()); + + filePosition += partSizeBytes; + } + + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest( + testBucket, key, res.getUploadId(), partETags); + + ionicS3Client.completeMultipartUpload(compRequest); + } catch (AmazonS3Exception ase) { + ionicS3Client.abortMultipartUpload( + new AbortMultipartUploadRequest(testBucket, key, res.getUploadId())); + fail("AmazonS3Exception during Multipart upload: " + ase.getErrorMessage()); + } + } + + private String extractIonicKeyIDFromMatdesc(String str) throws IOException { + Map map = new ObjectMapper().readValue(str, HashMap.class); + return map.get(IonicEncryptionMaterialsProvider.KEYIDKEY); + } + + private void waitUntilObjectExists(String bucket, String objectKey) throws TimeoutException, InterruptedException { + int attempts = 0; + while(s3Client.doesObjectExist(bucket, objectKey) == false && attempts < 5) { + attempts++; + if (attempts >= 5) { + throw new TimeoutException("Timed out waiting for uploaded object " + + objectKey + " to become available."); + } + Thread.sleep(1000); + } + } + +} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/SkipFailListener.java b/src/test/java/com/ionic/cloudstorage/awss3/SkipFailListener.java new file mode 100644 index 0000000..86eeb8f --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/SkipFailListener.java @@ -0,0 +1,23 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.junit.listen; + +import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunListener; + +public class SkipFailListener extends RunListener { + + static String failOnSkipProp = "failOnSkip"; + + public void testAssumptionFailure(Failure failure) { + String prop = System.getProperty(failOnSkipProp); + if (prop != null && prop.equalsIgnoreCase("true")) { + throw new RuntimeException("Test failed to meet preconditions."); + } + } + +} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/TestUtils.java b/src/test/java/com/ionic/cloudstorage/awss3/TestUtils.java new file mode 100644 index 0000000..695a0de --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/TestUtils.java @@ -0,0 +1,196 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.regions.DefaultAwsRegionProviderChain; +import com.amazonaws.SdkClientException; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.ionic.sdk.agent.Agent; +import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; +import com.ionic.sdk.error.AgentErrorModuleConstants; +import com.ionic.sdk.error.IonicException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import org.apache.commons.io.FileUtils; + + +public class TestUtils { + + protected static String testBucketEnv = "IONIC_S3_TEST_BUCKET"; + protected static String testBucketProp = "testBucket"; + protected static String testObjectKeyProp = "objectKey"; + protected static String testPayloadStringProp = "payloadString"; + protected static String testPersistorProp = "persistorPath"; + protected static String testFileSourceProp = "fileSource"; + protected static String testFileDestProp = "fileDest"; + protected static String testDirSourceProp = "dirSource"; + protected static String testDirDestProp = "dirDest"; + protected static String testPolicyDeniedProp = "policyDenied"; + + protected static String defaultPayload = "Hello World."; + + protected static String getTestBucket() { + String bucket = System.getProperty(testBucketProp); + if (bucket == null) { + bucket = System.getenv(testBucketEnv); + } + return bucket; + } + + protected static String getTestPayload() { + String string = System.getProperty(testPayloadStringProp); + if (string == null) { + string = defaultPayload; + } + return string; + } + + protected static String getTestObjectKey() { + return System.getProperty(testObjectKeyProp); + } + + protected static File getSourceFile() { + File file = null; + String filepath = System.getProperty(testFileSourceProp); + if (filepath != null) { + file = new File(filepath); + if (file.exists() == false) { + return null; + } + } + return file; + } + + protected static File getDestFile() { + File file = null; + String filepath = System.getProperty(testFileDestProp); + if (filepath != null) { + file = new File(filepath); + } + return file; + } + + protected static File getSourceDirectory() { + File file = null; + String filepath = System.getProperty(testDirSourceProp); + if (filepath != null) { + file = new File(filepath); + if (! (file.isDirectory() && file.exists())) { + return null; + } + } + return file; + } + + protected static File getDestDirectory() { + File file = null; + String filepath = System.getProperty(testDirDestProp); + if (filepath != null) { + file = new File(filepath); + } + return file; + } + + protected static boolean isProfilePolicyDenied() { + String denied = System.getProperty(testPolicyDeniedProp); + if (denied != null) { + return denied.equalsIgnoreCase("true"); + } + return false; + } + + protected static DeviceProfilePersistorPlainText getPersistor() throws IonicException { + DeviceProfilePersistorPlainText ptPersistor = null; + String ptPersitorPath = System.getProperty(testPersistorProp); + if (ptPersitorPath == null) { + ptPersitorPath = System.getProperty("user.home") + "/.ionicsecurity/profiles.pt"; + } + if (Files.exists(Paths.get(ptPersitorPath))) { + return new DeviceProfilePersistorPlainText(ptPersitorPath); + } else { + throw new IonicException(AgentErrorModuleConstants.ISAGENT_NO_DEVICE_PROFILE); + } + } + + protected static Agent getAgent() throws IonicException { + Agent agent = new Agent(); + agent.initialize(getPersistor()); + return agent; + } + + protected static IonicEncryptionMaterialsProvider getIEMP() throws IonicException { + return new IonicEncryptionMaterialsProvider(getPersistor()); + } + + protected static Boolean awsCredsAvailable() { + try { + DefaultAWSCredentialsProviderChain.getInstance().getCredentials(); + new DefaultAwsRegionProviderChain().getRegion(); + } catch (SdkClientException e) { + return false; + } + return true; + } + + protected static Boolean directoryContentsMatch(File a, File b, Boolean recurisve) throws IOException { + if (!(a.exists() && a.isDirectory() && b.exists() && b.isDirectory())){ + return false; + } + + HashMap aFileMap = fileMapFromArray(a.listFiles()); + HashMap bFileMap = fileMapFromArray(b.listFiles()); + + HashSet fileNames = new HashSet(); + fileNames.addAll(aFileMap.keySet()); + fileNames.addAll(bFileMap.keySet()); + + for (String key : fileNames) { + if (recurisve) { + if ((aFileMap.containsKey(key) && bFileMap.containsKey(key)) == false) { + return false; + } + if (aFileMap.get(key).isFile()) { + if (FileUtils.contentEquals(aFileMap.get(key), bFileMap.get(key)) == false) { + return false; + } + } else if (aFileMap.get(key).isDirectory()) { + if (directoryContentsMatch(aFileMap.get(key), bFileMap.get(key), true) == false) { + return false; + } + } else { + return false; + } + + } else { + if (aFileMap.containsKey(key) && aFileMap.get(key).isFile()) { + if (bFileMap.containsKey(key) == false) { + return false; + } else if (FileUtils.contentEquals(aFileMap.get(key), bFileMap.get(key)) == false) { + return false; + } + } else if (bFileMap.containsKey(key) && bFileMap.get(key).isFile()) { + return false; + } + } + } + return true; + } + + private static HashMap fileMapFromArray(File[] fileArray) { + HashMap map = new HashMap(); + for (int i = 0; i < fileArray.length; i++) { + map.put(fileArray[i].getName(), fileArray[i]); + } + return map; + } + +} diff --git a/src/test/java/com/ionic/cloudstorage/awss3/TransferManagerFunctionalTest.java b/src/test/java/com/ionic/cloudstorage/awss3/TransferManagerFunctionalTest.java new file mode 100644 index 0000000..c75c70a --- /dev/null +++ b/src/test/java/com/ionic/cloudstorage/awss3/TransferManagerFunctionalTest.java @@ -0,0 +1,182 @@ +/* + * (c) 2019 Ionic Security Inc. By using this code, I agree to the LICENSE included, as well as the + * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy + * (https://www.ionic.com/privacy-notice/). + */ + +package com.ionic.cloudstorage.awss3; + +import static org.junit.Assert.*; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.AmazonS3EncryptionClientBuilder; +import com.amazonaws.services.s3.model.EncryptionMaterials; +import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.transfer.Download; +import com.amazonaws.services.s3.transfer.MultipleFileDownload; +import com.amazonaws.services.s3.transfer.MultipleFileUpload; +import com.amazonaws.services.s3.transfer.Upload; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.ionic.sdk.error.IonicException; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import javax.crypto.spec.SecretKeySpec; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class TransferManagerFunctionalTest { + + private static IonicEncryptionMaterialsProvider iemp = null; + private static IonicS3EncryptionClient ionicS3Client = null; + private static AmazonS3 s3Client = null; + private static TransferManager transferManager = null; + + private static String testBucket = null; + + @BeforeClass + public static void setup() { + if (TestUtils.awsCredsAvailable()) { + try { + iemp = TestUtils.getIEMP(); + ionicS3Client = (IonicS3EncryptionClient)IonicS3EncryptionClientBuilder.standard() + .withEncryptionMaterials(iemp).build(); + transferManager = TransferManagerBuilder.standard().withS3Client(ionicS3Client).build(); + } catch (IonicException e) { + // Catch any IonicExceptions thrown during setup and null related objects so + // that dependent tests are each skipped during the preconditions check. + iemp = null; + ionicS3Client = null; + transferManager = null; + } + s3Client = AmazonS3ClientBuilder.defaultClient(); + } + testBucket = TestUtils.getTestBucket(); + } + + @Before + public void preconditions() { + Assume.assumeNotNull(iemp); + Assume.assumeNotNull(ionicS3Client); + Assume.assumeNotNull(s3Client); + Assume.assumeNotNull(transferManager); + Assume.assumeNotNull(testBucket); + } + + @Test + public void uploadAndDownloadFile() throws IOException, InterruptedException { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testUploadAndDownloadFile"; + } + + File sourceFile = TestUtils.getSourceFile(); + File destFile = TestUtils.getDestFile(); + Assume.assumeNotNull(sourceFile); + Assume.assumeNotNull(destFile); + + Upload upload = transferManager.upload(testBucket, key, sourceFile); + upload.waitForUploadResult(); + + Download download = transferManager.download(testBucket, key, destFile); + download.waitForCompletion(); + + assertTrue("Downloaded File did not match original File", + FileUtils.contentEquals(sourceFile, destFile)); + + S3Object obj = s3Client.getObject(testBucket, key); + + assertFalse("Test file was not encrypted at rest", + IOUtils.contentEquals(obj.getObjectContent(), FileUtils.openInputStream(sourceFile))); + } + + @Test + public void uploadAndDownloadFileComparison() throws IOException, InterruptedException { + String key = TestUtils.getTestObjectKey(); + if (key == null) { + key = "testUploadAndDownloadFileComparison"; + } + + byte[] rawBytes = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32}; + SecretKeySpec secretKey = new SecretKeySpec(rawBytes, "AES"); + EncryptionMaterials materials = new EncryptionMaterials(secretKey); + StaticEncryptionMaterialsProvider provider = new StaticEncryptionMaterialsProvider(materials); + AmazonS3 s3Client = AmazonS3EncryptionClientBuilder.standard().withEncryptionMaterials(provider).build(); + TransferManager s3transferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build(); + + File sourceFile = TestUtils.getSourceFile(); + File destFile = TestUtils.getDestFile(); + Assume.assumeNotNull(sourceFile); + Assume.assumeNotNull(destFile); + + Upload upload = s3transferManager.upload(testBucket, key, sourceFile); + upload.waitForUploadResult(); + + Download download = s3transferManager.download(testBucket, key, destFile); + download.waitForCompletion(); + + assertTrue("Downloaded File did not match original File", + FileUtils.contentEquals(sourceFile, destFile)); + + S3Object obj = s3Client.getObject(testBucket, key); + + assertTrue("File was not encrypted at rest", + IOUtils.contentEquals(obj.getObjectContent(), FileUtils.openInputStream(sourceFile))); + } + + @Test + public void uploadAndDownloadDirectory() throws IOException, InterruptedException { + String bucketDirectory = TestUtils.getTestObjectKey(); + if (bucketDirectory == null) { + bucketDirectory = "testUploadAndDownloadDirectory"; + } + + File sourceDir = TestUtils.getSourceDirectory(); + File destDir = TestUtils.getDestDirectory(); + Assume.assumeNotNull(sourceDir); + Assume.assumeNotNull(destDir); + + MultipleFileUpload upload = transferManager.uploadDirectory(testBucket, bucketDirectory, sourceDir, true); + upload.waitForCompletion(); + + MultipleFileDownload download = transferManager.downloadDirectory(testBucket, bucketDirectory, destDir); + download.waitForCompletion(); + + assertTrue("Downloaded directory contents did not match source directory", + TestUtils.directoryContentsMatch(sourceDir, new File(destDir, bucketDirectory), true)); + } + + @Test + public void uploadFileList() throws IOException, InterruptedException { + String bucketDirectory = TestUtils.getTestObjectKey(); + if (bucketDirectory == null) { + bucketDirectory = "testUploadFileList"; + } + + File sourceDir = TestUtils.getSourceDirectory(); + File destDir = TestUtils.getDestDirectory(); + Assume.assumeNotNull(sourceDir); + Assume.assumeNotNull(destDir); + + List fileList = Arrays.asList(sourceDir.listFiles()); + + MultipleFileUpload upload = transferManager + .uploadFileList(testBucket, bucketDirectory, sourceDir, fileList); + upload.waitForCompletion(); + + MultipleFileDownload download = transferManager.downloadDirectory(testBucket, bucketDirectory, destDir); + download.waitForCompletion(); + + assertTrue("Downloaded directory contents did not match source directory", + TestUtils.directoryContentsMatch(sourceDir, new File(destDir, bucketDirectory), false)); + } +} diff --git a/src/test/java/com/ionicsecurity/ipcs/awss3/IonicEncryptionMaterialsProviderTest.java b/src/test/java/com/ionicsecurity/ipcs/awss3/IonicEncryptionMaterialsProviderTest.java deleted file mode 100644 index d683b54..0000000 --- a/src/test/java/com/ionicsecurity/ipcs/awss3/IonicEncryptionMaterialsProviderTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * (c) 2018 Ionic Security Inc. - * By using this code, I agree to the LICENSE included, as well as the - * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy (https://www.ionic.com/privacy-notice/). - */ - -package com.ionicsecurity.ipcs.awss3; - -import java.io.IOException; -import java.nio.file.Paths; -import java.nio.file.InvalidPathException; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import javax.crypto.spec.SecretKeySpec; - -import com.amazonaws.services.s3.model.EncryptionMaterials; - -import com.ionic.sdk.agent.Agent; -import com.ionic.sdk.agent.AgentSdk; -import com.ionic.sdk.agent.key.KeyAttributesMap; -import com.ionic.sdk.agent.request.createkey.CreateKeysResponse; -import com.ionic.sdk.agent.request.getkey.GetKeysResponse; -import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; -import com.ionic.sdk.error.IonicException; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -public class IonicEncryptionMaterialsProviderTest extends TestCase { - - IonicEncryptionMaterialsProvider iemp; - DeviceProfilePersistorPlainText ptPersistor; - - public IonicEncryptionMaterialsProviderTest(String testName) throws InvalidPathException, IOException, IonicException { - super(testName); - setUp(); - } - - public static Test suite() { - return new TestSuite(IonicEncryptionMaterialsProviderTest.class); - } - - public void setUp() throws InvalidPathException, IOException, IonicException { - AgentSdk.initialize(null); - ptPersistor = new DeviceProfilePersistorPlainText(); - String sProfilePath = Paths.get(System.getProperty("user.home") + "/.ionicsecurity/profiles.pt").toFile().getCanonicalPath(); - ptPersistor.setFilePath(sProfilePath); - iemp = new IonicEncryptionMaterialsProvider(ptPersistor); - } - - public void testGetEncryptionMaterials() throws IonicException { - EncryptionMaterials encMat = iemp.getEncryptionMaterials(null); - assertTrue(encMat instanceof EncryptionMaterials); - - String ionicKeyId = encMat.getMaterialsDescription().get(IonicEncryptionMaterialsProvider.KEYIDKEY); - assertTrue(ionicKeyId != null); - - Agent agent = new Agent(); - GetKeysResponse.Key IonicKey = null; - agent.initialize(ptPersistor); - IonicKey = agent.getKey(ionicKeyId).getKeys().get(0); - assertTrue(IonicKey != null); - } - - public void testGetEncryptionMaterialsWithDesc() throws IonicException { - Map desc = new HashMap(); - Agent agent = new Agent(); - agent.initialize(ptPersistor); - - CreateKeysResponse.Key IonicKey = null; - ArrayList collection = new ArrayList(); - - String keyAttrMapKey = new String("IEMP_testGetEncryptionMaterialsWithDesc_KeyAttr"); - String keyAttrValue = new String("IEMP_testGetEncryptionMaterialsWithDesc_KeyAttrTestValue"); - collection.add(keyAttrValue); - KeyAttributesMap myKam = new KeyAttributesMap(); - myKam.put(keyAttrMapKey, collection); - IonicKey = agent.createKey(myKam).getFirstKey(); - desc.put(IonicEncryptionMaterialsProvider.KEYIDKEY, IonicKey.getId()); - - EncryptionMaterials encMat = iemp.getEncryptionMaterials(desc); - assertTrue(encMat instanceof EncryptionMaterials); - - SecretKeySpec awsKey = new SecretKeySpec(IonicKey.getKey(), 0, IonicKey.getKey().length, "AES"); - assertTrue(encMat.getSymmetricKey().equals(awsKey)); - } - - public void testSetandGetDefaultAttributes() throws IonicException { - KeyAttributesMap kam = new KeyAttributesMap(); - ArrayList collection = new ArrayList(); - collection.add("confidential"); - collection.add("secured"); - kam.put("privacy", collection); - - IonicEncryptionMaterialsProvider.setDefaultAttributes(kam); - assertTrue(IonicEncryptionMaterialsProvider.getDefaultAttributes().equals(kam)); - - EncryptionMaterials encMat = iemp.getEncryptionMaterials(null); - Agent agent = new Agent(); - GetKeysResponse.Key IonicKey = null; - agent.initialize(ptPersistor); - iemp = new IonicEncryptionMaterialsProvider(ptPersistor); - String ionicKeyId = encMat.getMaterialsDescription().get(IonicEncryptionMaterialsProvider.KEYIDKEY); - IonicKey = agent.getKey(ionicKeyId).getKeys().get(0); - KeyAttributesMap ionicKam = IonicKey.getAttributesMap(); - for (Iterator>> iter = kam.entrySet().iterator(); iter.hasNext();) { - Map.Entry> entry = iter.next(); - assertTrue(entry.getValue().equals(ionicKam.get(entry.getKey()))); - } - } -} diff --git a/src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientBuilderTest.java b/src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientBuilderTest.java deleted file mode 100644 index 958a4fd..0000000 --- a/src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientBuilderTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * (c) 2018 Ionic Security Inc. - * By using this code, I agree to the LICENSE included, as well as the - * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy (https://www.ionic.com/privacy-notice/). - */ - -package com.ionicsecurity.ipcs.awss3; - -import static org.junit.Assert.*; - -import java.util.Map; - -import org.junit.Test; - -import com.amazonaws.services.s3.AmazonS3Encryption; -import com.amazonaws.services.s3.model.EncryptionMaterials; -import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; - -public class IonicS3EncryptionClientBuilderTest { - - @Test - public void testBuilderWithNonIonicProvider() { - - class NonIonicEncryptionMaterialsProvider implements EncryptionMaterialsProvider { - @Override - public EncryptionMaterials getEncryptionMaterials(Map materialsDescription) { - return null; - } - - @Override - public EncryptionMaterials getEncryptionMaterials() { - return null; - } - - @Override - public void refresh() { - } - } - - IonicS3EncryptionClientBuilder iscb = new IonicS3EncryptionClientBuilder(); - - NonIonicEncryptionMaterialsProvider nIemp = new NonIonicEncryptionMaterialsProvider(); - iscb.setEncryptionMaterials(nIemp); - - try { - AmazonS3Encryption ise2 = iscb.build(); - fail("Non-Ionic EncryptionMaterialsProvider cannot successfully be used to instantiate a client"); - } catch (Exception e) { - // Successfully failed in try - } - } - - @Test - public void testBuilderWithandWithoutProvider() { - IonicS3EncryptionClientBuilder iscb = IonicS3EncryptionClientBuilder.standard(); - - try { - AmazonS3Encryption ise = iscb.build(); - fail("Expected an Illegal Argument Exception to be thrown"); - } catch (Exception e) { - assertTrue(e instanceof java.lang.IllegalArgumentException); - } - - try { - iscb.withEncryptionMaterials(IonicEncryptionMaterialsProvider.standard()); - } catch (Exception e) { - fail("No exception expected when IonicMaterialsProvider is supplied"); - } - } -} diff --git a/src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientTest.java b/src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientTest.java deleted file mode 100644 index 055e6fd..0000000 --- a/src/test/java/com/ionicsecurity/ipcs/awss3/IonicS3EncryptionClientTest.java +++ /dev/null @@ -1,368 +0,0 @@ -/* - * (c) 2018 Ionic Security Inc. - * By using this code, I agree to the LICENSE included, as well as the - * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy (https://www.ionic.com/privacy-notice/). - */ - -package com.ionicsecurity.ipcs.awss3; - -import static org.junit.Assert.*; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.Writer; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.IOUtils; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClient; -import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClientBuilder; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CreateBucketRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.UploadPartRequest; - -import com.ionic.sdk.agent.AgentSdk; -import com.ionic.sdk.agent.key.KeyAttributesMap; -import com.ionic.sdk.agent.request.createkey.CreateKeysRequest; -import com.ionic.sdk.device.profile.persistor.DeviceProfilePersistorPlainText; -import com.ionic.sdk.error.IonicException; - -import org.junit.Test; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -import org.junit.runner.RunWith; - -@RunWith(Parameterized.class) -public class IonicS3EncryptionClientTest { - - IonicS3EncryptionClient isec; - static String bucketName; - File uploadFile; - File largeUpload; - - static { - AmazonIdentityManagementClient idtyClient = (AmazonIdentityManagementClient) AmazonIdentityManagementClientBuilder - .defaultClient(); - bucketName = idtyClient.getUser().getUser().getUserName().toLowerCase() + "-ipcstrials"; - } - - @Parameters - public static List getEncryptionClients() throws IonicException { - AgentSdk.initialize(null); - IonicS3EncryptionClient isec; - - DeviceProfilePersistorPlainText ptPersistor = new DeviceProfilePersistorPlainText(); - String sProfilePath = System.getProperty("user.home") + "/.ionicsecurity/profiles.pt"; - - ptPersistor.setFilePath(sProfilePath); - - isec = (IonicS3EncryptionClient) IonicS3EncryptionClientBuilder.standard() - .withEncryptionMaterials(new IonicEncryptionMaterialsProvider(ptPersistor)).build(); - - List isecl = new ArrayList(); - - isecl.add(isec); - return isecl; - } - - public IonicS3EncryptionClientTest(IonicS3EncryptionClient inputIemp) throws URISyntaxException { - URL url1 = this.getClass().getResource("/testFile.txt"); - URI uri1 = new URI(url1.toString()); - this.uploadFile = new File(uri1.getPath()); - - URL url2 = this.getClass().getResource("/largeFile.txt"); - URI uri2 = new URI(url2.toString()); - this.largeUpload = new File(uri2.getPath()); - - isec = inputIemp; - } - - @Test - public void testPutAndGetObject() throws IOException { - try { - if (!(isec.doesBucketExist(bucketName))) { - isec.createBucket(new CreateBucketRequest(bucketName)); - } - } catch (AmazonServiceException ase) { - fail("AmazonServiceException while creating bucket: " + ase.getErrorMessage()); - } catch (AmazonClientException ace) { - fail("AmazonClientException while creating bucket: " + ace.getMessage()); - } - - String key = "firstTest"; - PutObjectRequest por = new PutObjectRequest(bucketName, key, uploadFile); - - try { - isec.putObject(por); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while putting: " + ase.getErrorMessage()); - } - - GetObjectRequest gor = new GetObjectRequest(bucketName, key); - S3Object obj = null; - - try { - obj = isec.getObject(gor); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while getting: " + ase.getErrorMessage()); - } - - assertEquals(obj.getKey(), key); - - S3ObjectInputStream content = obj.getObjectContent(); - FileInputStream originalStream = new FileInputStream(uploadFile); - - assertTrue(IOUtils.contentEquals(content, originalStream)); - originalStream.close(); - } - - @Test - public void testPutandGetViaRegularClient() throws IOException { - String key = "secondTest"; - S3Object theObject = null; - PutObjectRequest por = new PutObjectRequest(bucketName, key, uploadFile); - - try { - isec.putObject(por); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while putting for use with regular client: " + ase.getErrorMessage()); - } - - GetObjectRequest gor = new GetObjectRequest(bucketName, key); - AmazonS3 regularClient = AmazonS3ClientBuilder.defaultClient(); - - try { - theObject = regularClient.getObject(gor); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while getting: " + ase.getErrorMessage()); - } - - InputStream originalFileStream = new FileInputStream(uploadFile); - assertFalse(IOUtils.contentEquals(originalFileStream, theObject.getObjectContent())); - } - - @Test - public void testPutAndGetObjectMetadata() throws URISyntaxException { - URL url = this.getClass().getResource("/metaDataDest.txt"); - URI uri = new URI(url.toString()); - this.uploadFile = new File(uri.getPath()); - - File metaDest = new File(uri.getPath()); - String key = "secondTest"; - PutObjectRequest por = new PutObjectRequest(bucketName, key, uploadFile); - - Map inputMap = new HashMap(); - inputMap.put("first", "a"); - inputMap.put("second", "b"); - ObjectMetadata s3ObjectMetadata = new ObjectMetadata(); - s3ObjectMetadata.setUserMetadata(inputMap); - por.withMetadata(s3ObjectMetadata); - - try { - isec.putObject(por); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while putting with metadata: " + ase.getErrorMessage()); - } - - GetObjectRequest gor = new GetObjectRequest(bucketName, key); - ObjectMetadata meta = null; - - try { - meta = isec.getObject(gor, metaDest); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while getting with metadata: " + ase.getErrorMessage()); - } - - Map resultMap = meta.getUserMetadata(); - inputMap.forEach((k, v) -> assertTrue(v.equals(resultMap.get(k)))); - } - - @Test - public void testInitiateMultipartUpload() { - InitiateMultipartUploadRequest req = new InitiateMultipartUploadRequest(bucketName, "secondTest"); - - try { - InitiateMultipartUploadResult imur = isec.initiateMultipartUpload(req); - } catch (Exception e) { - fail("No exception is expected, but got: " + e.getMessage()); - } - } - - @Test - public void testMultipartRoundTripViaRegularClient() throws IOException { - String key = "multipartForRegularClientTest"; - S3Object theObject = null; - - try { - uploadMultipart(key); - } catch (AmazonS3Exception ase) { - fail("failed to upload multipart from regular client test: " + ase.getMessage()); - } - - GetObjectRequest gor = new GetObjectRequest(bucketName, key); - AmazonS3 regularClient = AmazonS3ClientBuilder.defaultClient(); - - try { - theObject = regularClient.getObject(gor); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while getting: " + ase.getErrorMessage()); - } - - InputStream originalFileStream = new FileInputStream(largeUpload); - assertFalse(IOUtils.contentEquals(originalFileStream, theObject.getObjectContent())); - - PrintWriter writer = new PrintWriter(largeUpload); - writer.print(""); - writer.close(); - } - - @Test - public void testMultipartRoundTripViaIonicClient() throws IOException { - String key = "multipartForIonicClientTest"; - S3Object theObject = null; - - try { - uploadMultipart(key); - } catch (AmazonS3Exception ase) { - fail("failed to upload multipart from ionic client test: " + ase.getMessage()); - } - - GetObjectRequest gor = new GetObjectRequest(bucketName, key); - - try { - theObject = isec.getObject(gor); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while getting: " + ase.getErrorMessage()); - } - - InputStream originalFileStream = new FileInputStream(largeUpload); - assertTrue(IOUtils.contentEquals(originalFileStream, theObject.getObjectContent())); - - PrintWriter writer = new PrintWriter(largeUpload); - writer.print(""); - writer.close(); - } - - public void uploadMultipart(String key) throws FileNotFoundException, IOException { - float totalChunks; - long partSizeBytes = 6 * 1024 * 1024; - List partETags = new ArrayList(); - - try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(largeUpload)))) { - for (int i = 0; i < 1500000; i++) { - writer.write("Repeating "); - } - } - - long contentLength = largeUpload.length(); - if (contentLength % partSizeBytes != 0) { - totalChunks = contentLength / partSizeBytes + 1; - } else { - totalChunks = contentLength / partSizeBytes; - } - - ObjectMetadata s3ObjectMetadata = new ObjectMetadata(); - - KeyAttributesMap attributes = new KeyAttributesMap(); - KeyAttributesMap mutableAttributes = new KeyAttributesMap(); - attributes.put("Attribute", Arrays.asList("Val1","Val2","Val3")); - mutableAttributes.put("Mutable-Attribute", Arrays.asList("Val1","Val2","Val3")); - - CreateKeysRequest.Key reqKey = new CreateKeysRequest.Key("", 1, attributes, mutableAttributes); - - InitiateMultipartUploadRequest req = new InitiateMultipartUploadRequest(bucketName, key, s3ObjectMetadata); - InitiateMultipartUploadResult res = isec.initiateMultipartUpload(req, reqKey); - - try { - long filePosition = 0; - for (int i = 1; filePosition < contentLength; i++) { - partSizeBytes = Math.min(partSizeBytes, (contentLength - filePosition)); - boolean isLast = i == totalChunks; - - UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucketName).withKey(key) - .withUploadId(res.getUploadId()).withPartNumber(i).withFileOffset(filePosition) - .withFile(largeUpload).withPartSize(partSizeBytes).withLastPart(isLast); - - partETags.add(isec.uploadPart(uploadRequest).getPartETag()); - - filePosition += partSizeBytes; - } - - CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, key, - res.getUploadId(), partETags); - - isec.completeMultipartUpload(compRequest); - } catch (AmazonS3Exception ase) { - isec.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, key, res.getUploadId())); - fail("AmazonS3Exception during Multipart upload: " + ase.getErrorMessage()); - } - } - - @Test - public void testPutAndGetWithCreatRequestKey() throws IOException { - try { - if (!(isec.doesBucketExist(bucketName))) { - isec.createBucket(new CreateBucketRequest(bucketName)); - } - } catch (AmazonServiceException ase) { - fail("AmazonServiceException while creating bucket: " + ase.getErrorMessage()); - } catch (AmazonClientException ace) { - fail("AmazonClientException while creating bucket: " + ace.getMessage()); - } - - String key = "attributesTest"; - PutObjectRequest por = new PutObjectRequest(bucketName, key, uploadFile); - KeyAttributesMap attributes = new KeyAttributesMap(); - KeyAttributesMap mutableAttributes = new KeyAttributesMap(); - attributes.put("Attribute", Arrays.asList("Val1","Val2","Val3")); - mutableAttributes.put("Mutable-Attribute", Arrays.asList("Val1","Val2","Val3")); - - CreateKeysRequest.Key reqKey = new CreateKeysRequest.Key("", 1, attributes, mutableAttributes); - - try { - isec.putObject(por, reqKey); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while putting: " + ase.getErrorMessage()); - } - - GetObjectRequest gor = new GetObjectRequest(bucketName, key); - IonicS3EncryptionClient.IonicKeyS3ObjectPair pair = null; - - try { - pair = isec.getObjectAndKey(gor); - } catch (AmazonS3Exception ase) { - fail("AmazonS3Exception while getting: " + ase.getErrorMessage()); - } - - assertEquals(pair.getKey().getAttributesMap().get("Attribute"), attributes.get("Attribute")); - assertTrue(pair.getKey().getMutableAttributesMap().equals(mutableAttributes)); - } -} diff --git a/src/test/java/com/ionicsecurity/ipcs/awss3/TransferManagerTest.java b/src/test/java/com/ionicsecurity/ipcs/awss3/TransferManagerTest.java deleted file mode 100644 index 1e3b646..0000000 --- a/src/test/java/com/ionicsecurity/ipcs/awss3/TransferManagerTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * (c) 2018 Ionic Security Inc. - * By using this code, I agree to the LICENSE included, as well as the - * Terms & Conditions (https://dev.ionic.com/use.html) and the Privacy Policy (https://www.ionic.com/privacy-notice/). - */ - -package com.ionicsecurity.ipcs.awss3; - -import static org.junit.Assert.*; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.Writer; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; - -import org.apache.commons.io.IOUtils; -import org.junit.Test; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClient; -import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClientBuilder; -import com.amazonaws.services.s3.transfer.Download; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; - -public class TransferManagerTest { - - static String bucketName; - - static { - AmazonIdentityManagementClient idtyClient = - (AmazonIdentityManagementClient) AmazonIdentityManagementClientBuilder.defaultClient(); - bucketName = idtyClient.getUser().getUser().getUserName().toLowerCase() + "-ipcstrials"; - } - - @Test - public void testBasicTransferRoundTrip() - throws AmazonClientException, InterruptedException, IOException, URISyntaxException { - URL url = this.getClass().getResource("/testFile.txt"); - URI uri = new URI(url.toString()); - File uploadFile = new File(uri.getPath()); - assertTrue(roundTrip(uploadFile)); - } - - @Test - public void testMultipartTransferRoundTrip() - throws FileNotFoundException, IOException, - AmazonClientException, InterruptedException, URISyntaxException { - URL url = this.getClass().getResource("/largeFile.txt"); - URI uri = new URI(url.toString()); - File largeFile = new File(uri.getPath()); - - try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(largeFile)))) { - for (int i = 0; i < 15000; i++) { - writer.write("Repeating "); - } - } - - assertTrue(roundTrip(largeFile)); - - PrintWriter writer = new PrintWriter(largeFile); - writer.print(""); - writer.close(); - } - - public boolean roundTrip(File uploadFile) - throws IOException, AmazonClientException, InterruptedException, URISyntaxException { - - TransferManager xfrmngr = TransferManagerBuilder.standard() - .withS3Client(IonicS3EncryptionClientBuilder.defaultClient()).build(); - - try { - Upload xfer = xfrmngr.upload(bucketName, "transferTest1", uploadFile); - xfer.waitForCompletion(); - } catch (AmazonServiceException e) { - fail("Transfer Manager failed an upload: " + e.getErrorMessage()); - } - - URL url = this.getClass().getResource("/transferTarget1.txt"); - URI uri = new URI(url.toString()); - File downloadTarget = new File(uri.getPath()); - - try { - Download xfer = xfrmngr.download(bucketName, "transferTest1", downloadTarget); - xfer.waitForCompletion(); - } catch (AmazonServiceException e) { - fail("Transfer Manager failed an download: " + e.getErrorMessage()); - } - xfrmngr.shutdownNow(); - - InputStream originalFileStream = new FileInputStream(uploadFile); - InputStream downloadStream = new FileInputStream(downloadTarget); - boolean result = IOUtils.contentEquals(originalFileStream, downloadStream); - - PrintWriter writer = new PrintWriter(downloadTarget); - writer.print(""); - writer.close(); - return result; - } -} diff --git a/src/test/resources/largeFile.txt b/src/test/resources/largeFile.txt deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/resources/metaDataDest.txt b/src/test/resources/metaDataDest.txt deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/resources/testFile.txt b/src/test/resources/testFile.txt deleted file mode 100644 index 9944a9f..0000000 --- a/src/test/resources/testFile.txt +++ /dev/null @@ -1 +0,0 @@ -This is a test file \ No newline at end of file diff --git a/src/test/resources/transferTarget1.txt b/src/test/resources/transferTarget1.txt deleted file mode 100644 index e69de29..0000000