diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java index 7ad03981e..8905c00bd 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/S3ObjectList.java @@ -45,4 +45,4 @@ public void setS3Objects(final List s3Objects) { this.s3Objects = s3Objects; } -} \ No newline at end of file +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java index e7366eb67..c9c4fa96b 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/CompleteMultipartUpload.java @@ -31,6 +31,10 @@ public CompleteMultipartUpload() { //Pass } + public CompleteMultipartUpload(final List parts) { + this.parts = parts; + } + public List getParts() { return parts; } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java index 6bfe38252..6bf68fc0c 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/multipart/Part.java @@ -29,6 +29,11 @@ public Part() { //Pass } + public Part(final int partNumber, final String eTag) { + this.partNumber = partNumber; + this.eTag = eTag; + } + public int getPartNumber() { return partNumber; } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java index 337d362b8..1862f4dfb 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java @@ -15,10 +15,7 @@ package com.spectralogic.ds3client; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; -import com.google.common.collect.TreeMultimap; +import com.google.common.collect.*; import com.spectralogic.ds3client.commands.*; import com.spectralogic.ds3client.commands.decorators.PutFolderRequest; import com.spectralogic.ds3client.commands.spectrads3.*; @@ -27,7 +24,11 @@ import com.spectralogic.ds3client.models.*; import com.spectralogic.ds3client.models.Objects; import com.spectralogic.ds3client.models.bulk.Ds3Object; +import com.spectralogic.ds3client.models.bulk.PartialDs3Object; import com.spectralogic.ds3client.models.common.Credentials; +import com.spectralogic.ds3client.models.common.Range; +import com.spectralogic.ds3client.models.multipart.CompleteMultipartUpload; +import com.spectralogic.ds3client.models.multipart.Part; import com.spectralogic.ds3client.networking.FailedRequestException; import com.spectralogic.ds3client.networking.FailedRequestUsingMgmtPortException; import com.spectralogic.ds3client.networking.HttpVerb; @@ -35,7 +36,6 @@ import com.spectralogic.ds3client.utils.ResourceUtils; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertTrue; import java.io.IOException; import java.net.URISyntaxException; @@ -73,6 +73,16 @@ public class Ds3Client_Test { private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + private final static String IDS_REQUEST_PAYLOAD = "id1id2id3"; + private final static ImmutableList IDS_REQUEST_PAYLOAD_LIST = ImmutableList.of("id1", "id2", "id3"); + + private static final String SIMPLE_OBJECT_REQUEST_PAYLOAD = ""; + private static final List SIMPLE_OBJECT_LIST = Arrays.asList( + new Ds3Object("file1"), + new Ds3Object("file2"), + new Ds3Object("file3") + ); + @Before public void setTimeZone() { DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -531,7 +541,7 @@ MasterObjectList performRestCall(final Ds3Client client, final String bucket, fi throws IOException; } - public void runBulkTest(final BulkCommand command, final BulkTestDriver driver) throws IOException { + private void runBulkTest(final BulkCommand command, final BulkTestDriver driver) throws IOException { final List objects = Arrays.asList( new Ds3Object("file1", 256), new Ds3Object("file2", 1202), @@ -1146,19 +1156,348 @@ public void createFolderWithNoSlash() throws IOException { @Test public void clearSuspectBlobAzureTargetsSpectraS3Test() throws IOException { - final String expectedRequestContent = "id1id2id3"; + MockNetwork + .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_azure_target", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .clearSuspectBlobAzureTargetsSpectraS3(new ClearSuspectBlobAzureTargetsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void clearSuspectBlobPoolsSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_pool", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .clearSuspectBlobPoolsSpectraS3(new ClearSuspectBlobPoolsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void clearSuspectBlobS3TargetsSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_s3_target", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .clearSuspectBlobS3TargetsSpectraS3(new ClearSuspectBlobS3TargetsSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void clearSuspectBlobTapesSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_tape", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .clearSuspectBlobTapesSpectraS3(new ClearSuspectBlobTapesSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void markSuspectBlobAzureTargetsAsDegradedSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_azure_target", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .markSuspectBlobAzureTargetsAsDegradedSpectraS3(new MarkSuspectBlobAzureTargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void markSuspectBlobDs3TargetsAsDegradedSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_ds3_target", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .markSuspectBlobDs3TargetsAsDegradedSpectraS3(new MarkSuspectBlobDs3TargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void markSuspectBlobPoolsAsDegradedSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_pool", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .markSuspectBlobPoolsAsDegradedSpectraS3(new MarkSuspectBlobPoolsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void markSuspectBlobS3TargetsAsDegradedSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_s3_target", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .markSuspectBlobS3TargetsAsDegradedSpectraS3(new MarkSuspectBlobS3TargetsAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void markSuspectBlobTapesAsDegradedSpectraS3Test() throws IOException { + MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/suspect_blob_tape", new HashMap(), IDS_REQUEST_PAYLOAD) + .returning(204, "") + .asClient() + .markSuspectBlobTapesAsDegradedSpectraS3(new MarkSuspectBlobTapesAsDegradedSpectraS3Request(IDS_REQUEST_PAYLOAD_LIST)); + } + + @Test + public void completeMultiPartUploadTest() throws IOException { + final String expectedRequestContent = "17a112844c1a2327e617f530cb06dccf827162e29f4e40da7f521d0794b57770ba"; + final String expectedResponse = "http://my-server/bucketName/objectbucketNameobjectb54357faf0632cce46e942fa68356b38"; + + final List parts = new ArrayList<>(); + parts.add(new Part(1, "7a112844c1a2327e617f530cb06dccf8")); + parts.add(new Part(2, "7162e29f4e40da7f521d0794b57770ba")); + + final String bucketName = "bucketName"; + final String objectName = "object"; + final String uploadId = "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"; + final String etag = "b54357faf0632cce46e942fa68356b38"; + final String location = "http://my-server/bucketName/object"; + + final CompleteMultipartUpload completeMultipartUpload = new CompleteMultipartUpload(parts); + + final Map queryParams = new HashMap<>(); + queryParams.put("upload_id", uploadId); + + final Map responseHeaders = new HashMap<>(); + responseHeaders.put("etag", etag); + + final CompleteMultiPartUploadResponse response = MockNetwork + .expecting(HttpVerb.POST, "/bucketName/object", queryParams, expectedRequestContent) + .returning(200, expectedResponse, responseHeaders) + .asClient() + .completeMultiPartUpload(new CompleteMultiPartUploadRequest( + bucketName, + objectName, + completeMultipartUpload, + uploadId)); + + assertThat(response.getCompleteMultipartUploadResult().getLocation(), is(location)); + assertThat(response.getCompleteMultipartUploadResult().getBucket(), is(bucketName)); + assertThat(response.getCompleteMultipartUploadResult().getKey(), is(objectName)); + assertThat(response.getCompleteMultipartUploadResult().getETag(), is(etag)); + } + + @Test + public void putMultiPartUploadPartTest() throws IOException { + final String requestContent = "this is the part content"; + final String bucketName = "bucketName"; + final String objectName = "object"; + final int partNumber = 2; + final String uploadId = "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"; + final String eTag = "b54357faf0632cce46e942fa68356b38"; + + final Map queryParams = new HashMap<>(); + queryParams.put("part_number", String.valueOf(partNumber)); + queryParams.put("upload_id", uploadId); + + final Map responseHeaders = new HashMap<>(); + responseHeaders.put("etag", eTag); + + MockNetwork + .expecting(HttpVerb.PUT, "/bucketName/object", queryParams, requestContent) + .returning(200, "", responseHeaders) + .asClient() + .putMultiPartUploadPart(new PutMultiPartUploadPartRequest( + bucketName, + objectName, + new ByteArraySeekableByteChannel(requestContent.getBytes()), + partNumber, requestContent.getBytes().length, + uploadId)); + } + + @Test + public void getBulkJobSpectraS3WithMixedPayloadTest() throws IOException { + final List objects = Arrays.asList( + new Ds3Object("file1", 256), + new PartialDs3Object("file2", new Range(1202, 1402)), + new Ds3Object("file3", 2523) + ); + + final String bucketName = "bulkTest"; + + final String expectedRequestPayload = ""; + + final String xmlResponse = ""; + + final Map queryParams = new HashMap<>(); + queryParams.put("operation", "start_bulk_get"); + + final GetBulkJobSpectraS3Response response = MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, expectedRequestPayload) + .returning(200, xmlResponse) + .asClient() + .getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(bucketName, objects)); - final List ids = new ArrayList<>(); - ids.add("id1"); - ids.add("id2"); - ids.add("id3"); + final List objectListList = response.getMasterObjectList().getObjects(); + assertThat(objectListList.size(), is(1)); + + final List objectList = objectListList.get(0).getObjects(); + assertThat(objectList.size(), is(3)); + + assertObjectEquals(objectList.get(0), "file2", 1202); + assertObjectEquals(objectList.get(1), "file1", 256); + assertObjectEquals(objectList.get(2), "file3", 2523); + } + + + @Test + public void verifyBulkJobSpectraS3Test() throws IOException { + final List objects = Arrays.asList( + new Ds3Object("file1"), + new PartialDs3Object("file2", new Range(1202, 1402)), + new Ds3Object("file3") + ); + + final String bucketName = "bulkTest"; + + final String expectedRequestPayload = ""; + + final String xmlResponse = ""; final Map queryParams = new HashMap<>(); + queryParams.put("operation", "start_bulk_verify"); + + final VerifyBulkJobSpectraS3Response response = MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, expectedRequestPayload) + .returning(200, xmlResponse) + .asClient() + .verifyBulkJobSpectraS3(new VerifyBulkJobSpectraS3Request(bucketName, objects)); + + final List objectListList = response.getMasterObjectListResult().getObjects(); + assertThat(objectListList.size(), is(1)); + + final List objectList = objectListList.get(0).getObjects(); + assertThat(objectList.size(), is(3)); + + assertObjectEquals(objectList.get(0), "file2", 1202); + assertObjectEquals(objectList.get(1), "file1", 256); + assertObjectEquals(objectList.get(2), "file3", 2523); + } + + @Test + public void getPhysicalPlacementForObjectsTest() throws IOException { + final String responsePayload = ""; + final String bucketName = "BucketName"; + + final Map queryParams = new HashMap<>(); + queryParams.put("operation", "get_physical_placement"); + + final GetPhysicalPlacementForObjectsSpectraS3Response response = MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD) + .returning(200, responsePayload) + .asClient() + .getPhysicalPlacementForObjectsSpectraS3(new GetPhysicalPlacementForObjectsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST)); + + assertThat(response.getPhysicalPlacementResult().getAzureTargets(), is(nullValue())); + assertThat(response.getPhysicalPlacementResult().getDs3Targets(), is(nullValue())); + assertThat(response.getPhysicalPlacementResult().getPools(), is(nullValue())); + assertThat(response.getPhysicalPlacementResult().getS3Targets(), is(nullValue())); + assertThat(response.getPhysicalPlacementResult().getTapes(), is(nullValue())); + } + + @Test + public void getPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException { + final String responsePayload = ""; + final String bucketName = "BucketName"; + + final Map queryParams = new HashMap<>(); + queryParams.put("operation", "get_physical_placement"); + queryParams.put("full_details", null); + + final GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Response response = MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD) + .returning(200, responsePayload) + .asClient() + .getPhysicalPlacementForObjectsWithFullDetailsSpectraS3(new GetPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST)); + + assertThat(response.getBulkObjectListResult().getObjects().size(), is(1)); + } + + @Test + public void verifyPhysicalPlacementForObjectsTest() throws IOException { + final String responsePayload = "false10000t1false48d30ecb-84f1-4721-9832-7aa165a1dd7776343269-c32a-4cb0-aec4-57a9dccce6eaPENDING_INSPECTIONfalse20000LTO5false"; + final String bucketName = "BucketName"; + + final Map queryParams = new HashMap<>(); + queryParams.put("operation", "verify_physical_placement"); + + final VerifyPhysicalPlacementForObjectsSpectraS3Response response = MockNetwork + .expecting(HttpVerb.GET, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD) + .returning(200, responsePayload) + .asClient() + .verifyPhysicalPlacementForObjectsSpectraS3(new VerifyPhysicalPlacementForObjectsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST)); + + assertThat(response.getPhysicalPlacementResult().getAzureTargets(), is(nullValue())); + assertThat(response.getPhysicalPlacementResult().getDs3Targets(), is(nullValue())); + assertThat(response.getPhysicalPlacementResult().getPools(), is(nullValue())); + assertThat(response.getPhysicalPlacementResult().getTapes().size(), is(1)); + } + + @Test + public void verifyPhysicalPlacementForObjectsWithFullDetailsTest() throws IOException { + final String responsePayload = "false10000t1false5a7bb215-4aff-4806-b217-5fe01ade6a2c2e5b25fc-546e-45b0-951e-8f3d80bb7823PENDING_INSPECTIONfalse20000LTO5false"; + final String bucketName = "BucketName"; + + final Map queryParams = new HashMap<>(); + queryParams.put("operation", "verify_physical_placement"); + queryParams.put("full_details", null); + + final VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Response response = MockNetwork + .expecting(HttpVerb.GET, "/_rest_/bucket/" + bucketName, queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD) + .returning(200, responsePayload) + .asClient() + .verifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3(new VerifyPhysicalPlacementForObjectsWithFullDetailsSpectraS3Request(bucketName, SIMPLE_OBJECT_LIST)); + + assertThat(response.getBulkObjectListResult().getObjects().size(), is(1)); + } + + @Test + public void ejectStorageDomainBlobsTest() throws IOException { + final String bucketId = "BucketId"; + final String storageDomainId = "StorageDomainId"; + + final Map queryParams = new HashMap<>(); + queryParams.put("operation", "eject"); + queryParams.put("blobs", null); + queryParams.put("bucket_id", bucketId); + queryParams.put("storage_domain_id", storageDomainId); MockNetwork - .expecting(HttpVerb.DELETE, "/_rest_/suspect_blob_azure_target", queryParams, expectedRequestContent) + .expecting(HttpVerb.PUT, "/_rest_/tape", queryParams, SIMPLE_OBJECT_REQUEST_PAYLOAD) .returning(204, "") .asClient() - .clearSuspectBlobAzureTargetsSpectraS3(new ClearSuspectBlobAzureTargetsSpectraS3Request(ids)); + .ejectStorageDomainBlobsSpectraS3(new EjectStorageDomainBlobsSpectraS3Request(bucketId, SIMPLE_OBJECT_LIST, storageDomainId)); + } + + @Test + public void replicatePutJobTest() throws IOException { + final String responsePayload = ""; + final String requestPayload = "This is the request payload content"; + final String bucketName = "BucketName"; + + final Map queryParams = new HashMap<>(); + queryParams.put("operation", "start_bulk_put"); + queryParams.put("replicate", null); + + final ReplicatePutJobSpectraS3Response response = MockNetwork + .expecting(HttpVerb.PUT, "/_rest_/bucket/" + bucketName, queryParams, requestPayload) + .returning(200, responsePayload) + .asClient() + .replicatePutJobSpectraS3(new ReplicatePutJobSpectraS3Request(bucketName, requestPayload)); + + assertThat(response.getMasterObjectListResult().getObjects().size(), is(1)); + } + + @Test + public void getBlobPersistenceTest() throws IOException { + final String responsePayload = "This is the response payload content"; + final String requestPayload = "This is the request payload content"; + + final Map queryParams = new HashMap<>(); + + final GetBlobPersistenceSpectraS3Response response = MockNetwork + .expecting(HttpVerb.GET, "/_rest_/blob_persistence", queryParams, requestPayload) + .returning(200, responsePayload) + .asClient() + .getBlobPersistenceSpectraS3(new GetBlobPersistenceSpectraS3Request(requestPayload)); + + assertThat(response.getStringResult(), is(responsePayload)); } } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java index 6448b9078..371395bba 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/models/ModelParsing_Test.java @@ -92,17 +92,11 @@ public void completeMultipartUpload_ToString_Test() { "20c78aef83f66abc1fa1e8477f296d394" + ""; - final Part part1 = new Part(); - part1.setPartNumber(1); - part1.seteTag("a54357aff0632cce46d942af68356b38"); + final ImmutableList parts = ImmutableList.of( + new Part(1, "a54357aff0632cce46d942af68356b38"), + new Part(2, "0c78aef83f66abc1fa1e8477f296d394")); - final Part part2 = new Part(); - part2.setPartNumber(2); - part2.seteTag("0c78aef83f66abc1fa1e8477f296d394"); - - final ImmutableList parts = ImmutableList.of(part1, part2); - final CompleteMultipartUpload input = new CompleteMultipartUpload(); - input.setParts(parts.asList()); + final CompleteMultipartUpload input = new CompleteMultipartUpload(parts.asList()); final String result = XmlOutput.toXml(input);