diff --git a/.travis.yml b/.travis.yml index 5b850af23..8d5b0b1e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,4 @@ language: java jdk: - oraclejdk8 - - oraclejdk7 - - openjdk7 script: ./gradlew -S clean jar ds3-sdk:test ds3-utils:test ds3-metadata:test diff --git a/build.gradle b/build.gradle index fcb760a76..1fd5a771b 100644 --- a/build.gradle +++ b/build.gradle @@ -13,45 +13,61 @@ * **************************************************************************** */ +buildscript { + ext.kotlin_version = '1.1.4-2' + + repositories { + mavenCentral() + } + + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + } +} + + allprojects { group = 'com.spectralogic.ds3' - version = '3.5.1' + version = '3.5.2' } subprojects { apply plugin: 'maven' apply plugin: 'java' + apply plugin: 'kotlin' apply plugin: 'findbugs' - sourceCompatibility = JavaVersion.VERSION_1_7 + sourceCompatibility = JavaVersion.VERSION_1_8 repositories { mavenCentral() mavenLocal() } dependencies { - compile 'org.slf4j:slf4j-api:1.7.22' - testCompile ('org.mockito:mockito-core:1.10.19') { + compile "org.slf4j:slf4j-api:$slf4jVersion" + testCompile ("org.mockito:mockito-core:$mockitoVersion") { exclude group: 'org.hamcrest' } - testCompile 'junit:junit:4.12' - testCompile 'org.slf4j:slf4j-simple:1.7.22' + testCompile "junit:junit:$junitVersion" + testCompile "org.slf4j:slf4j-simple:$slf4jVersion" } } task wrapper(type: Wrapper) { - gradleVersion = '3.5' + gradleVersion = '4.1' } project(':ds3-sdk') { dependencies { + compile project(':ds3-interfaces') compile project(':ds3-utils') } } project(':ds3-metadata') { dependencies { + compile project(':ds3-interfaces') compile project(':ds3-utils') } } diff --git a/ds3-metadata/build.gradle b/ds3-metadata/build.gradle index 9d45f587d..6002069a0 100644 --- a/ds3-metadata/build.gradle +++ b/ds3-metadata/build.gradle @@ -16,7 +16,7 @@ buildscript { repositories { jcenter() } dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3' + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1' } } @@ -26,14 +26,19 @@ shadowJar { relocate 'org.apache', 'ds3metafatjar.org.apache' relocate 'com.google', 'ds3metafatjar.com.google' relocate 'com.sun.jna', 'ds3metafatjar.net.java.dev.jna' + relocate 'org.jetbrains', 'ds3metafatjar.org.jetbrains' + relocate 'org.intellij', 'ds3metafatjar.org.intellij' + relocate 'org.codehaus', 'ds3metafatjar.org.codehaus' dependencies { - exclude(dependency('org.hamcrest:hamcrest-library:1.3')) - exclude(dependency('org.mockito:mockito-core:1.10.19')) - exclude(dependency('junit:junit:4.12')) - exclude(dependency('org.slf4j:slf4j-api:1.7.22')) - exclude(dependency('org.slf4j:slf4j-simple:1.7.22')) - exclude(dependency('org.apache.commons:commons-lang3:3.0')) + exclude(project(":ds3-interfaces")) // this is being excluded since it must be used with the sdk, which already has this dependency included + exclude(project(":ds3-utils")) // this is being excluded since it must be used with the sdk, which already has this dependency included + exclude(dependency("org.hamcrest:hamcrest-library:$hamcrestVersion")) + exclude(dependency("org.mockito:mockito-core:$mockitoVersion")) + exclude(dependency("junit:junit:$junitVersion")) + exclude(dependency("org.slf4j:slf4j-api:$slf4jVersion")) + exclude(dependency("org.slf4j:slf4j-simple:$slf4jVersion")) + exclude(dependency("org.apache.commons:commons-lang3:$commonslangVersion")) } } @@ -42,10 +47,10 @@ artifacts { } dependencies { - compile group: 'net.java.dev.jna', name: 'jna-platform', version: '4.2.2' - compile group: 'net.java.dev.jna', name: 'jna', version: '4.2.2' - compile 'commons-io:commons-io:2.4' - compile 'org.apache.httpcomponents:httpclient:4.5.1' + compile group: 'net.java.dev.jna', name: 'jna-platform', version: "$jnaVersion" + compile group: 'net.java.dev.jna', name: 'jna', version: "$jnaVersion" + compile "commons-io:commons-io:$commonsioVersion" + compile "org.apache.httpcomponents:httpclient:$httpclientVersion" - testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0' + testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "$commonslangVersion" } diff --git a/ds3-sdk-integration/build.gradle b/ds3-sdk-integration/build.gradle index 56fcfed45..362cf2907 100644 --- a/ds3-sdk-integration/build.gradle +++ b/ds3-sdk-integration/build.gradle @@ -14,9 +14,9 @@ */ dependencies { - compile 'commons-codec:commons-codec:1.10' - compile 'junit:junit:4.12' - testCompile 'org.hamcrest:hamcrest-library:1.3' - testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0' + compile "commons-codec:commons-codec:$commonscodecVersion" + compile "junit:junit:$junitVersion" + testCompile "org.hamcrest:hamcrest-library:$hamcrestVersion" + testCompile group: 'org.apache.commons', name: 'commons-lang3', version: "$commonslangVersion" } diff --git a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java index 166f73532..069eafbde 100644 --- a/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java +++ b/ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java @@ -20,6 +20,7 @@ import com.spectralogic.ds3client.commands.DeleteBucketRequest; import com.spectralogic.ds3client.commands.DeleteObjectRequest; import com.spectralogic.ds3client.commands.spectrads3.GetSystemInformationSpectraS3Request; +import com.spectralogic.ds3client.helpers.DeleteBucket; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; import com.spectralogic.ds3client.helpers.channelbuilders.PrefixAdderObjectChannelBuilder; import com.spectralogic.ds3client.models.Contents; @@ -39,7 +40,7 @@ import static org.junit.Assume.assumeThat; -public class Util { +public final class Util { private static final Logger LOG = LoggerFactory.getLogger(Util.class); public static final String RESOURCE_BASE_NAME = "books/"; public static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"}; @@ -103,22 +104,11 @@ public static void loadBookTestDataWithPrefix(final Ds3Client client, final Stri } public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException { - final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); - - final Iterable objects = helpers.listObjects(bucketName); - for(final Contents contents : objects) { - client.deleteObject(new DeleteObjectRequest(bucketName, contents.getKey())); - } - - client.deleteBucket(new DeleteBucketRequest(bucketName)); + Ds3ClientHelpers.wrap(client).deleteBucket(bucketName); } public static void deleteBucketContents(final Ds3Client client, final String bucketName) throws IOException { final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); - - final Iterable objects = helpers.listObjects(bucketName); - for(final Contents contents : objects) { - client.deleteObject(new DeleteObjectRequest(bucketName, contents.getKey())); - } + DeleteBucket.INSTANCE.deleteBucketContents(helpers, bucketName); } } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java index 34d4af2e7..f6cf1f665 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/test/helpers/TempStorageUtil.java @@ -30,7 +30,7 @@ * and partition for use in the integration tests to avoid error if the BP does not currently * have a partition available for running the unit tests. */ -public class TempStorageUtil { +public final class TempStorageUtil { private static final String DATA_POLICY_NAME = "_dp"; private static final String STORAGE_DOMAIN_NAME = "_sd"; diff --git a/ds3-sdk-samples/build.gradle b/ds3-sdk-samples/build.gradle index e0b2a6709..f71a1040a 100644 --- a/ds3-sdk-samples/build.gradle +++ b/ds3-sdk-samples/build.gradle @@ -18,5 +18,5 @@ apply plugin: 'application' mainClassName = 'com.spectralogic.ds3client.samples.PartialObjectGetExample' dependencies { - compile 'org.slf4j:slf4j-simple:1.7.22' + compile "org.slf4j:slf4j-simple:$slf4jVersion" } diff --git a/ds3-sdk/build.gradle b/ds3-sdk/build.gradle index 87f7c6925..5b74d8b55 100644 --- a/ds3-sdk/build.gradle +++ b/ds3-sdk/build.gradle @@ -19,7 +19,7 @@ import java.nio.file.Path buildscript { repositories { jcenter() } dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3' + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1' } } @@ -27,17 +27,25 @@ apply plugin: 'com.github.johnrengelman.shadow' shadowJar { relocate 'com.google', 'ds3fatjar.com.google' - relocate 'org.apache', 'ds3fatjar.org.apache' + relocate 'org.jetbrains', 'ds3fatjar.org.jetbrains' + relocate 'org.intellij', 'ds3fatjar.org.intellij' relocate 'org.codehaus', 'ds3fatjar.org.codehaus' + relocate 'kotlin', 'ds3fatjar.kotlin' + relocate 'edu.umd', 'ds3fatjar.edu.emd' + relocate 'net.jcip', 'ds3fatjar.net.jcip' + relocate 'com.ctc', 'ds3fatjar.com.ctc' + relocate 'org.apache', 'ds3fatjar.org.apache' relocate 'com.fasterxml', 'ds3fatjar.com.fasterxml' dependencies { - exclude(dependency('org.hamcrest:hamcrest-library:1.3')) - exclude(dependency('org.mockito:mockito-core:1.10.19')) - exclude(dependency('junit:junit:4.12')) - exclude(dependency('org.slf4j:slf4j-api:1.7.22')) - exclude(dependency('org.slf4j:slf4j-simple:1.7.22')) - exclude(dependency('org.apache.commons:commons-lang3:3.0')) + exclude(dependency("org.hamcrest:hamcrest-library:$hamcrestVersion")) + exclude(dependency("org.mockito:mockito-core:$mockitoVersion")) + exclude(dependency("junit:junit:$junitVersion")) + exclude(dependency("org.slf4j:slf4j-api:$slf4jVersion")) + exclude(dependency("org.slf4j:slf4j-simple:$slf4jVersion")) + exclude(dependency("org.apache.commons:commons-lang3:$commonslangVersion")) } + + mergeServiceFiles() } artifacts { @@ -78,10 +86,12 @@ jar { jar.dependsOn genConfigProperties dependencies { - compile 'org.apache.httpcomponents:httpclient:4.5.1' + compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + compile "org.apache.httpcomponents:httpclient:$httpclientVersion" + compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$jacksonVersion" + compile "com.google.guava:guava:$guavaVersion" compile 'org.codehaus.woodstox:woodstox-core-asl:4.4.1' - compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.7.1' - compile 'com.google.guava:guava:20.0' compile 'com.google.code.findbugs:annotations:3.0.1' - testCompile 'org.hamcrest:hamcrest-library:1.3' + testCompile "org.hamcrest:hamcrest-library:$hamcrestVersion" } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java index faae9155a..4097d8425 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/interfaces/AbstractRequest.java @@ -36,7 +36,7 @@ private static RequestHeaders buildDefaultHeaders() { @Override public String getContentType() { - return ContentType.APPLICATION_XML.toString(); + return ContentType.APPLICATION_XML.getMimeType(); } @Override diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java index 1818dfe05..f01d4f4dc 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java @@ -509,6 +509,12 @@ public abstract Iterable listObjects(final String bucket, final String */ public abstract Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException; + /** + * + * @param bucket + */ + public abstract void deleteBucket(final String bucket) throws IOException; + /** * Returns an Iterable of {@link Ds3Object} that have a prefix added. */ @@ -526,20 +532,12 @@ public abstract Iterable listObjects(final String bucket, final String @SafeVarargs public final Iterable toDs3Iterable(final Iterable objects, final Predicate... filters) { - FluentIterable fluentIterable = FluentIterable.from(objects).filter(new com.google.common.base.Predicate() { - @Override - public boolean apply(@Nullable final Contents input) { - return input != null; - } - }); + FluentIterable fluentIterable = FluentIterable.from(objects).filter(input -> input != null); if (filters != null) { for (final Predicate filter : filters) { - fluentIterable = fluentIterable.filter(new com.google.common.base.Predicate() { - @Override - public boolean apply(@Nullable final Contents input) { - return filter == null || filter.test(input); // do not filter anything if filter is null - } + fluentIterable = fluentIterable.filter(input -> { + return filter == null || filter.test(input); // do not filter anything if filter is null }); } } @@ -586,4 +584,9 @@ public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInD * Creates a folder in the specified bucket */ public abstract void createFolder(final String bucketName, final String folderName) throws IOException; + + /** + * Returns the client being wrapped + */ + public abstract Ds3Client getClient(); } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java index 8db5373f1..0d562443b 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java @@ -15,12 +15,9 @@ package com.spectralogic.ds3client.helpers; -import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Lists; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.commands.*; import com.spectralogic.ds3client.commands.spectrads3.*; @@ -40,20 +37,17 @@ import com.spectralogic.ds3client.models.common.Range; import com.spectralogic.ds3client.networking.FailedRequestException; import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel; +import com.spectralogic.ds3client.utils.collections.StreamWrapper; import com.spectralogic.ds3client.utils.collections.LazyIterable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.Collection; -import java.util.List; import java.util.UUID; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; class Ds3ClientHelpersImpl extends Ds3ClientHelpers { private final static Logger LOG = LoggerFactory.getLogger(Ds3ClientHelpersImpl.class); @@ -147,7 +141,7 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket, final TransferStrategyBuilder transferStrategyBuilder) throws IOException { - final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(bucket, Lists.newArrayList(objectsToWrite)) + final PutBulkJobSpectraS3Request request = new PutBulkJobSpectraS3Request(bucket, objectsToWrite) .withPriority(options.getPriority()) .withAggregating(options.isAggregating()) .withForce(options.isForce()) @@ -166,19 +160,19 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket, } @Override - public Job startWriteJob(final TransferStrategy transferStrategy) + public Ds3ClientHelpers.Job startWriteJob(final TransferStrategy transferStrategy) throws IOException { return new WriteJobImpl(transferStrategy); } @Override - public Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite) throws IOException { + public Ds3ClientHelpers.Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite) throws IOException { return startWriteJobUsingStreamedBehavior(bucket, objectsToWrite, WriteJobOptions.create()); } @Override - public Job startWriteJobUsingStreamedBehavior(final String bucket, + public Ds3ClientHelpers.Job startWriteJobUsingStreamedBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException @@ -192,12 +186,12 @@ public Job startWriteJobUsingStreamedBehavior(final String bucket, } @Override - public Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite) throws IOException { + public Ds3ClientHelpers.Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite) throws IOException { return startWriteJobUsingRandomAccessBehavior(bucket, objectsToWrite, WriteJobOptions.create()); } @Override - public Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException { + public Ds3ClientHelpers.Job startWriteJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToWrite, final WriteJobOptions options) throws IOException { Preconditions.checkNotNull(options, "options may not be null."); final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder(); @@ -212,10 +206,8 @@ public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) - throws IOException - { - final List objects = Lists.newArrayList(objectsToRead); + public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objects, final ReadJobOptions options) + throws IOException { final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request; @@ -230,13 +222,13 @@ public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objects, final ReadJobOptions options) { + private GetBulkJobSpectraS3Request makeGetBulkJobSpectraS3Request(final String bucket, final Iterable objects, final ReadJobOptions options) { return new GetBulkJobSpectraS3Request(bucket, objects) .withPriority(options.getPriority()) .withName(options.getName()); } - private Ds3ClientHelpers.Job innerStartReadJob(final List objects, + private Ds3ClientHelpers.Job innerStartReadJob(final Iterable objects, final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request, final TransferStrategyBuilder transferStrategyBuilder) throws IOException @@ -255,21 +247,19 @@ private Ds3ClientHelpers.Job innerStartReadJob(final List objects, } @Override - public Job startReadJob(final TransferStrategy transferStrategy) throws IOException { + public Ds3ClientHelpers.Job startReadJob(final TransferStrategy transferStrategy) throws IOException { return new ReadJobImpl(transferStrategy); } @Override - public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead) throws IOException { + public Ds3ClientHelpers.Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead) throws IOException { return startReadJobUsingStreamedBehavior(bucket, objectsToRead, ReadJobOptions.create()); } @Override - public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException { + public Ds3ClientHelpers.Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable objects, final ReadJobOptions options) throws IOException { Preconditions.checkNotNull(options, "options may not be null."); - final List objects = Lists.newArrayList(objectsToRead); - final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, options); getBulkJobSpectraS3Request.withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.IN_ORDER); @@ -281,16 +271,14 @@ public Job startReadJobUsingStreamedBehavior(final String bucket, final Iterable } @Override - public Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead) throws IOException { + public Ds3ClientHelpers.Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead) throws IOException { return startReadJobUsingRandomAccessBehavior(bucket, objectsToRead, ReadJobOptions.create()); } @Override - public Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws IOException { + public Ds3ClientHelpers.Job startReadJobUsingRandomAccessBehavior(final String bucket, final Iterable objects, final ReadJobOptions options) throws IOException { Preconditions.checkNotNull(options, "options may not be null."); - final List objects = Lists.newArrayList(objectsToRead); - final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = makeGetBulkJobSpectraS3Request(bucket, objects, options); getBulkJobSpectraS3Request.withChunkClientProcessingOrderGuarantee(JobChunkClientProcessingOrderGuarantee.NONE); @@ -319,7 +307,7 @@ private Ds3ClientHelpers.Job innerStartReadAllJob(final String bucket, final Rea return this.startReadJob(bucket, makeBlobList(bucket), options); } - private final Iterable makeBlobList(final String bucket) throws IOException { + private Iterable makeBlobList(final String bucket) throws IOException { final Iterable contentsList = listObjects(bucket); return toDs3Iterable(contentsList, FolderNameFilter.filter()); @@ -362,8 +350,7 @@ public Ds3ClientHelpers.Job recoverWriteJob(final UUID jobId) throws IOException /** * Verifies that the specified job is active. If the job is not active, then a * JobRecoveryNotActiveException is thrown. - * @throws IOException - * @throws JobRecoveryNotActiveException + * @throws JobRecoveryNotActiveException This is thrown when the job is no longer active when a recovery is attempted */ private void innerVerifyJobActive(final UUID jobId) throws IOException, JobRecoveryNotActiveException { try { @@ -421,7 +408,7 @@ public Ds3ClientHelpers.Job recoverReadJob(final UUID jobId) throws IOException, final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() .withMasterObjectList(masterObjectList) - .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())); + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())); return new ReadJobImpl(transferStrategyBuilder); } @@ -433,28 +420,28 @@ private MasterObjectList masterObjectListForGetJob(final UUID jobId) throws IOEx } @Override - public Job recoverReadJobsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException { + public Ds3ClientHelpers.Job recoverReadJobsingStreamedBehavior(final UUID jobId) throws IOException, JobRecoveryException { innerVerifyJobActive(jobId); final MasterObjectList masterObjectList = masterObjectListForGetJob(jobId); final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() .withMasterObjectList(masterObjectList) - .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())) .usingStreamedTransferBehavior(); return new ReadJobImpl(transferStrategyBuilder); } @Override - public Job recoverReadJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException { + public Ds3ClientHelpers.Job recoverReadJobUsingRandomAccessBehavior(final UUID jobId) throws IOException, JobRecoveryException { innerVerifyJobActive(jobId); final MasterObjectList masterObjectList = masterObjectListForGetJob(jobId); final TransferStrategyBuilder transferStrategyBuilder = makeTransferStrategyBuilder() .withMasterObjectList(masterObjectList) - .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())) + .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(), ImmutableMultimap.of())) .usingRandomAccessTransferBehavior(); return new ReadJobImpl(transferStrategyBuilder); @@ -514,20 +501,12 @@ public Iterable listObjects(final String bucket, final String keyPrefi @Override public Iterable listObjects(final String bucket, final String keyPrefix, final String nextMarker, final int maxKeys, final int retries) { - return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, GetBucketKeyLoaderFactory.contentsFunction)); + return new LazyIterable<>(new GetBucketKeyLoaderFactory<>(client, bucket, keyPrefix, DEFAULT_DELIMITER, nextMarker, maxKeys, retries, GetBucketKeyLoaderFactory.contentsFunction)); } @Override public Iterable listObjectsForDirectory(final Path directory) throws IOException { - final ImmutableList.Builder objects = ImmutableList.builder(); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { - objects.add(new Ds3Object(directory.relativize(file).toString().replace("\\", "/"), Files.size(file))); - return FileVisitResult.CONTINUE; - } - }); - return objects.build(); + return FileTreeWalker.INSTANCE.walk(directory); } @Override @@ -545,31 +524,26 @@ public Iterable remoteListDirectory(final String bucket, final St } public Iterable remoteListDirectory(final String bucket, final String keyPrefix, final String delimiter, final String nextMarker, final int maxKeys) throws IOException { - return new LazyIterable<>(new GetBucketKeyLoaderFactory(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, GetBucketKeyLoaderFactory.getFileSystemKeysFunction)); + return new LazyIterable<>(new GetBucketKeyLoaderFactory<>(client, bucket, keyPrefix, delimiter, nextMarker, maxKeys, DEFAULT_LIST_OBJECTS_RETRIES, GetBucketKeyLoaderFactory.getFileSystemKeysFunction)); + } + + @Override + public void deleteBucket(final String bucket) throws IOException { + DeleteBucket.INSTANCE.deleteBucket(this, bucket); } + @Override public Iterable addPrefixToDs3ObjectsList(final Iterable objectsList, final String prefix) { - final FluentIterable objectIterable = FluentIterable.from(objectsList); + final Stream objectIterable = StreamSupport.stream(objectsList.spliterator(), false); - return objectIterable.transform(new Function() { - @Nullable - @Override - public Ds3Object apply(@Nullable final Ds3Object object) { - return new Ds3Object( prefix + object.getName(), object.getSize()); - } - }); + return StreamWrapper.wrapStream(objectIterable.map(obj -> new Ds3Object(prefix + obj.getName(), obj.getSize()))); } + @Override public Iterable removePrefixFromDs3ObjectsList(final Iterable objectsList, final String prefix) { - final FluentIterable objectIterable = FluentIterable.from(objectsList); + final Stream objectIterable = StreamSupport.stream(objectsList.spliterator(), false); - return objectIterable.transform(new Function() { - @Nullable - @Override - public Ds3Object apply(@Nullable final Ds3Object object) { - return new Ds3Object(stripLeadingPath(object.getName(), prefix), object.getSize()); - } - }); + return StreamWrapper.wrapStream(objectIterable.map(obj -> new Ds3Object(stripLeadingPath(obj.getName(), prefix), obj.getSize()))); } /** @@ -615,6 +589,11 @@ public void createFolder(final String bucketName, final String folderName) throw client.putObject(folderRequest); } + @Override + public Ds3Client getClient() { + return client; + } + /** * Ensures that a folder names ends with a trailing forward slash. This prevents the * accidental creation of zero-length files when attempting to create a folder. diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java index ab41b2127..4d7ee0386 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/StrategyUtils.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; -import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.JobNode; import com.spectralogic.ds3client.models.Objects; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java index a8f6ff9c3..9501f2220 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/strategy/blobstrategy/GetSequentialBlobStrategy.java @@ -25,8 +25,6 @@ import com.spectralogic.ds3client.models.BulkObject; import com.spectralogic.ds3client.models.MasterObjectList; import com.spectralogic.ds3client.models.Objects; -import org.slf4j.LoggerFactory; -import org.slf4j.Logger; import javax.annotation.Nullable; import java.io.IOException; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java index 89f6776ff..6296dd04a 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/util/PartialObjectHelpers.java @@ -78,10 +78,9 @@ static ImmutableList dedupRanges(final ImmutableList rangesForBlob Range currentRange = null; Range nextRange; - final UnmodifiableIterator rangeIterator = sortedRanges.iterator(); - while (rangeIterator.hasNext()) { - nextRange = rangeIterator.next(); + for (final Range sortedRange : sortedRanges) { + nextRange = sortedRange; if (currentRange == null) { // This will only be called on the first iteration of the loop currentRange = nextRange; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java index 9a34369f4..e6f5d6565 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java @@ -21,11 +21,15 @@ import com.spectralogic.ds3client.models.Priority; import com.spectralogic.ds3client.models.JobChunkClientProcessingOrderGuarantee; import com.spectralogic.ds3client.models.WriteOptimization; +import com.spectralogic.ds3client.utils.collections.StreamWrapper; + +import java.util.stream.Stream; +import java.util.stream.StreamSupport; @JacksonXmlRootElement(localName = "Objects") public class Ds3ObjectList { @JsonProperty("Object") - private Iterable objects; + private Stream objects; @JacksonXmlProperty(isAttribute = true, namespace = "", localName = "Priority") private Priority priority; @@ -40,15 +44,15 @@ public Ds3ObjectList() { } public Ds3ObjectList(final Iterable objects) { - this.objects = objects; + this.objects = StreamSupport.stream(objects.spliterator(), false); } public Iterable getObjects() { - return objects; + return StreamWrapper.wrapStream(this.objects); } public void setObjects(final Iterable objects) { - this.objects = objects; + this.objects = StreamSupport.stream(objects.spliterator(), false); } public Priority getPriority() { diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java index 346973197..ab2f78f90 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/serializer/XmlOutput.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.spectralogic.ds3client.models.bulk.Ds3ObjectList; import java.io.IOException; @@ -42,6 +43,8 @@ public final class XmlOutput { module = new JacksonXmlModule(); module.setDefaultUseWrapper(false); mapper = new XmlMapper(module); + + mapper.registerModule(new Jdk8Module()); final SimpleFilterProvider filterProvider = new SimpleFilterProvider().setFailOnUnknownId(false); mapper.setFilterProvider(filterProvider); if (isProductionBuild()) { diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java index 40655806a..3c3d6f19b 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/JobUtils.java @@ -24,7 +24,6 @@ import com.spectralogic.ds3client.models.*; import java.io.IOException; -import java.security.SignatureException; import java.util.ArrayList; import java.util.List; import java.util.Set; diff --git a/ds3-interfaces/build.gradle b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/StreamWrapper.java similarity index 54% rename from ds3-interfaces/build.gradle rename to ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/StreamWrapper.java index ac8ff43ef..8fdb7adf1 100644 --- a/ds3-interfaces/build.gradle +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/collections/StreamWrapper.java @@ -13,30 +13,31 @@ * **************************************************************************** */ -buildscript { - repositories { jcenter() } - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3' - } -} +package com.spectralogic.ds3client.utils.collections; -apply plugin: 'com.github.johnrengelman.shadow' +import org.jetbrains.annotations.NotNull; -dependencies { - compile 'commons-io:commons-io:2.4' -} +import java.util.Iterator; +import java.util.stream.Stream; + +/** + * Wraps a Stream as an Iterable + */ +public class StreamWrapper implements Iterable { -shadowJar { - relocate 'org.apache', 'ds3fatjar.org.apache' - dependencies { - exclude(dependency('org.hamcrest:hamcrest-library:1.3')) - exclude(dependency('org.mockito:mockito-core:1.10.19')) - exclude(dependency('junit:junit:4.12')) - exclude(dependency('org.slf4j:slf4j-api:1.7.22')) - exclude(dependency('org.slf4j:slf4j-simple:1.7.22')) + private final Stream stream; + + public static Iterable wrapStream(final Stream stream) { + return new StreamWrapper<>(stream); } -} -artifacts { - archives shadowJar + private StreamWrapper(final Stream stream) { + this.stream = stream; + } + + @NotNull + @Override + public Iterator iterator() { + return stream.iterator(); + } } diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/DeleteBucket.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/DeleteBucket.kt new file mode 100644 index 000000000..8d03ff69e --- /dev/null +++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/DeleteBucket.kt @@ -0,0 +1,38 @@ +/* + * ****************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers + +import com.spectralogic.ds3client.commands.DeleteBucketRequest +import com.spectralogic.ds3client.commands.DeleteObjectsRequest +import com.spectralogic.ds3client.utils.collections.asIterable +import com.spectralogic.ds3client.utils.collections.take + +object DeleteBucket { + fun deleteBucket(helpers: Ds3ClientHelpers, bucket: String) { + deleteBucketContents(helpers, bucket) + helpers.client.deleteBucket(DeleteBucketRequest(bucket)) + } + + fun deleteBucketContents(helpers: Ds3ClientHelpers, bucket: String) { + val client = helpers.client + val objIterator = helpers.listObjects(bucket).iterator() + + while(objIterator.hasNext()) { + val subIterator = objIterator.take(1_000) + client.deleteObjects(DeleteObjectsRequest(bucket, subIterator.asIterable())) + } + } +} \ No newline at end of file diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker.kt new file mode 100644 index 000000000..18b1d4270 --- /dev/null +++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker.kt @@ -0,0 +1,35 @@ +/* + * ****************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers + +import com.spectralogic.ds3client.models.bulk.Ds3Object +import com.spectralogic.ds3client.utils.collections.StreamWrapper +import java.nio.file.Files +import java.nio.file.Path + +object FileTreeWalker { + /** + * Walks a file system starting at {@param startDir} and returns a Lazy Iterable with all the + * files represented as Ds3Objects. + */ + fun walk(startDir: Path) : Iterable { + return StreamWrapper.wrapStream(Files.walk(startDir).map { + Ds3Object(startDir.relativize(it).toString().replace("\\", "/"), Files.size(it)) + }.filter { + !it.name.isNullOrEmpty() + }) + } +} diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/IterableExtensions.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/IterableExtensions.kt new file mode 100644 index 000000000..223dc15d7 --- /dev/null +++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/IterableExtensions.kt @@ -0,0 +1,24 @@ +/* + * ****************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils.collections + +fun Iterator.asIterable(): Iterable { + return IterableWrapper(this) +} + +private class IterableWrapper constructor(private val iterator: Iterator): Iterable { + override fun iterator(): Iterator = iterator +} diff --git a/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator.kt b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator.kt new file mode 100644 index 000000000..5b570c89e --- /dev/null +++ b/ds3-sdk/src/main/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator.kt @@ -0,0 +1,43 @@ +/* + * ****************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils.collections + +class WindowedIterator constructor(private val iterator: Iterator, private val windowSize: Int) : Iterator by iterator { + + init { + if (windowSize <= 0) throw IllegalArgumentException("windowSize must be larger than 0") + } + + private var count = 0 + + override fun next(): T { + if (!hasNext()) throw NoSuchElementException("There are no new items to return") + + count++ + return iterator.next() + } + + override fun hasNext(): Boolean { + return if (count >= windowSize) { + false + } else { + iterator.hasNext() + } + } +} + +fun Iterator.take(n: Int): Iterator = WindowedIterator(this, n) + diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java index 389a30d3f..6f1dfc6c2 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/ConnectionFixture.java @@ -18,7 +18,7 @@ import com.spectralogic.ds3client.models.common.Credentials; import com.spectralogic.ds3client.networking.ConnectionDetails; -public class ConnectionFixture { +public final class ConnectionFixture { public static ConnectionDetails getConnection() { return getConnection(8080); diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java index 93b9413fd..8eba5e4ef 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java @@ -22,7 +22,6 @@ import com.spectralogic.ds3client.commands.*; import com.spectralogic.ds3client.commands.spectrads3.*; import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException; -import com.spectralogic.ds3client.exceptions.FolderNameMissingTrailingForwardSlash; import com.spectralogic.ds3client.models.*; import com.spectralogic.ds3client.models.Objects; import com.spectralogic.ds3client.models.bulk.Ds3Object; @@ -50,6 +49,7 @@ import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class Ds3Client_Test { private static final UUID MASTER_OBJECT_LIST_JOB_ID = UUID.fromString("1a85e743-ec8f-4789-afec-97e587a26936"); @@ -504,24 +504,12 @@ public void getObjectWithMetaData() throws IOException { @Test public void createPutJobSpectraS3() throws IOException { - this.runBulkTest(BulkCommand.PUT, new BulkTestDriver() { - @Override - public MasterObjectList performRestCall(final Ds3Client client, final String bucket, final List objects) - throws IOException { - return client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList(); - } - }); + this.runBulkTest(BulkCommand.PUT, (client, bucket, objects) -> client.putBulkJobSpectraS3(new PutBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList()); } @Test public void createGetJobSpectraS3() throws IOException { - this.runBulkTest(BulkCommand.GET, new BulkTestDriver() { - @Override - public MasterObjectList performRestCall(final Ds3Client client, final String bucket, final List objects) - throws IOException { - return client.getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList(); - } - }); + this.runBulkTest(BulkCommand.GET, (client, bucket, objects) -> client.getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(bucket, objects)).getMasterObjectList()); } private interface BulkTestDriver { @@ -529,7 +517,7 @@ MasterObjectList performRestCall(final Ds3Client client, final String bucket, fi throws IOException; } - public void runBulkTest(final BulkCommand command, final BulkTestDriver driver) throws IOException { + private void runBulkTest(final BulkCommand command, final Ds3Client_Test.BulkTestDriver driver) throws IOException { final List objects = Arrays.asList( new Ds3Object("file1", 256), new Ds3Object("file2", 1202), @@ -926,7 +914,7 @@ public void testGettingDefaultUserAgent() { final String userAgent = newClient.getConnectionDetails().getUserAgent(); final String[] userAgentFields = userAgent.split("-"); - assertThat(userAgentFields.length >= 2, is(true)); + assertThat(userAgentFields.length, is(greaterThanOrEqualTo(2))); // look for a pattern like 3.4.0, but leave open the possibility of a string like 3.4.0-SNAPSHOT final Pattern matchPattern = Pattern.compile("\\d+\\.\\d+\\.\\d+"); diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java index 35e73bba3..ec6e384da 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/MockNetwork.java @@ -92,7 +92,7 @@ public MockNetwork returning( public MockNetwork returning( final int statusCode, final String responseContent) { - return returning(statusCode, responseContent, new HashMap()); + return returning(statusCode, responseContent, new HashMap<>()); } public Ds3Client asClient() { diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java index 1eadaa633..9ed9e4e43 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/SeekableByteChannelInputStream_Test.java @@ -100,7 +100,7 @@ private void readAndVerify(final InputStream inputStream, final byte[] expectedB final byte[] resultBuffer = new byte[expectedBytes.length]; final byte[] buffer = new byte[10]; int position = 0; - int bytesRead = 0; + int bytesRead; while (0 < (bytesRead = inputStream.read(buffer, 5, 5))) { for (int i = 0; i < bytesRead; i++) { resultBuffer[position] = buffer[5 + i]; diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java index 28071aec8..eafa594f0 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/commands/parsers/utils/ResponseParserUtils_Test.java @@ -47,7 +47,7 @@ public void getRequestId_WithoutRequestId_Test() { @Test public void getRequestId_EmptyHeaders_Test() { - final String result = ResponseParserUtils.getRequestId(new MockedHeaders(ImmutableMap.of())); + final String result = ResponseParserUtils.getRequestId(new MockedHeaders(ImmutableMap.of())); assertThat(result, is(nullValue())); } } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index abe990803..3181f7302 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -109,7 +109,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { public void testReadObjectsWithFailedGet() throws IOException, ParseException { final Ds3Client ds3Client = mock(Ds3Client.class); - Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client); + Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client); final GetBulkJobSpectraS3Response buildBulkGetResponse = buildBulkGetResponse(); Mockito.when(ds3Client.getBulkJobSpectraS3(hasChunkOrdering(JobChunkClientProcessingOrderGuarantee.NONE))).thenReturn(buildBulkGetResponse); @@ -204,7 +204,7 @@ public void testWriteObjectsWithFailedPut() throws IOException, ParseException { final ConnectionDetails details = mock(ConnectionDetails.class); Mockito.when(details.getEndpoint()).thenReturn("localhost"); - Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client); + Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client); Mockito.when(ds3Client.getConnectionDetails()).thenReturn(details); final PutBulkJobSpectraS3Response buildBulkPutResponse = buildBulkPutResponse(); @@ -424,7 +424,7 @@ private static Ds3Client buildDs3ClientForBulk() throws IOException, .thenReturn(jobChunksResponse2) .thenReturn(jobChunksResponse3); - Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client); + Mockito.when(ds3Client.newForNode(Matchers.any())).thenReturn(ds3Client); Mockito.when(ds3Client.getConnectionDetails()).thenReturn(details); return ds3Client; } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java index 695666d4c..2e449cd00 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java @@ -34,7 +34,7 @@ import static org.mockito.Matchers.argThat; -public class RequestMatchers { +public final class RequestMatchers { public static GetBulkJobSpectraS3Request hasChunkOrdering(final JobChunkClientProcessingOrderGuarantee chunkOrdering) { return argThat(new TypeSafeMatcher() { @Override @@ -225,7 +225,7 @@ public boolean matches(final Object argument) { && (marker == null ? null == getBucketRequest.getMarker() : marker.equals(getBucketRequest.getMarker())); - + } }); } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java index 84e22db71..eb242aae6 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/channelbuilders/StreamObjectPutter.java @@ -27,7 +27,7 @@ public StreamObjectPutter(final byte[] buf) { } @Override - public InputStream buildInputStream(String key) { + public InputStream buildInputStream(final String key) { return new ByteArrayInputStream(buf); } } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java index 229b6e4f1..1dfbbdbde 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java @@ -167,21 +167,21 @@ public void escapeSpacesInQueryParam() throws MalformedURLException { @Test public void getPortBack() throws MalformedURLException { final URL url = new URL("http://localhost:8080/path"); - int port = NetUtils.getPort(url); + final int port = NetUtils.getPort(url); assertTrue(port == 8080); } @Test public void getHttpsDefaultPort() throws MalformedURLException { final URL url = new URL("https://localhost/path"); - int port = NetUtils.getPort(url); + final int port = NetUtils.getPort(url); assertTrue(port == 443); } @Test public void getHttpDefaultPort() throws MalformedURLException { final URL url = new URL("http://localhost/path"); - int port = NetUtils.getPort(url); + final int port = NetUtils.getPort(url); assertTrue(port == 80); } diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java index 3052ef645..6e3ff31d0 100644 --- a/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/utils/Signature_Test.java @@ -17,7 +17,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Multimap; import com.spectralogic.ds3client.networking.HttpVerb; import com.spectralogic.ds3client.models.common.Credentials; import com.spectralogic.ds3client.models.common.SignatureDetails; @@ -32,7 +31,6 @@ public class Signature_Test { /** * Example taken from the AWS S3 documentation site: http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader - * @throws SignatureException */ @Test public void getSignature() throws SignatureException { @@ -47,7 +45,6 @@ public void getSignature() throws SignatureException { /** * Example taken from the AWS S3 documentation site: http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader - * @throws SignatureException */ @Test public void putSignature() throws SignatureException { diff --git a/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker_Test.kt b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker_Test.kt new file mode 100644 index 000000000..ac1d0d7c6 --- /dev/null +++ b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/helpers/FileTreeWalker_Test.kt @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers + +import com.google.common.collect.ImmutableList +import com.spectralogic.ds3client.models.bulk.Ds3Object +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +import org.hamcrest.CoreMatchers.* +import org.hamcrest.MatcherAssert.assertThat + +class FileTreeWalker_Test { + + @get:Rule + val tempDir = TemporaryFolder() + + @Test + fun basicFileList() { + tempDir.newFile("bar") + tempDir.newFile("baz") + + val walk = FileTreeWalker.walk(tempDir.root.toPath()) + + val listBuilder = ImmutableList.Builder() + + walk.forEach { + listBuilder.add(it) + } + + val fileList = listBuilder.build() + + assertThat(fileList.size, `is`(2)) + + } +} \ No newline at end of file diff --git a/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator_Test.kt b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator_Test.kt new file mode 100644 index 000000000..3a0951ac0 --- /dev/null +++ b/ds3-sdk/src/test/kotlin/com/spectralogic/ds3client/utils/collections/WindowedIterator_Test.kt @@ -0,0 +1,76 @@ +/* + * ****************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils.collections + +import com.google.common.collect.Lists +import org.junit.Test + +import org.hamcrest.CoreMatchers.* +import org.hamcrest.MatcherAssert.assertThat + +class WindowedIterator_Test { + @Test + fun iterateWholeCollection() { + val list = Lists.newArrayList("value", "hi", "anothervalue") + + val iterator = list.iterator().take(Int.MAX_VALUE) + + var count = 0 + iterator.forEach { + count++ + } + + assertThat(count, `is`(3)) + } + + @Test + fun iteratePart() { + val list = Lists.newArrayList("value", "hi", "anothervalue") + + val iterator = list.iterator().take(1) + + var count = 0 + iterator.forEach { + count++ + } + + assertThat(count, `is`(1)) + } + + @Test + fun iterateAfterFirstIteration() { + val list = Lists.newArrayList("value", "hi", "anothervalue") + + val parentIterator = list.iterator() + val firstIterator = parentIterator.take(1) + + var firstCount = 0 + firstIterator.forEach { + firstCount++ + } + + assertThat(firstCount, `is`(1)) + + val secondIterator = parentIterator.take(Int.MAX_VALUE) + + var secondCount = 0 + secondIterator.forEach { + secondCount++ + } + + assertThat(secondCount, `is`(2)) + } +} \ No newline at end of file diff --git a/ds3-utils/build.gradle b/ds3-utils/build.gradle index 6b89d6230..f8f974efa 100644 --- a/ds3-utils/build.gradle +++ b/ds3-utils/build.gradle @@ -14,34 +14,9 @@ * **************************************************************************** */ -buildscript { - repositories { jcenter() } - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3' - } -} - -apply plugin: 'com.github.johnrengelman.shadow' - -shadowJar { - relocate 'org.apache', 'ds3utilfatjar.org.apache' - relocate 'com.google', 'ds3utilfatjar.com.google' - - dependencies { - exclude(dependency('org.hamcrest:hamcrest-library:1.3')) - exclude(dependency('org.mockito:mockito-core:1.10.19')) - exclude(dependency('junit:junit:4.12')) - exclude(dependency('org.slf4j:slf4j-api:1.7.22')) - exclude(dependency('org.slf4j:slf4j-simple:1.7.22')) - } -} - -artifacts { - archives shadowJar -} - dependencies { - compile 'commons-codec:commons-codec:1.10' - compile 'com.google.guava:guava:20.0' + compile "commons-codec:commons-codec:$commonscodecVersion" + compile "commons-io:commons-io:$commonsioVersion" + compile "com.google.guava:guava:$guavaVersion" } diff --git a/ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java b/ds3-utils/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java similarity index 100% rename from ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java rename to ds3-utils/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java diff --git a/ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java b/ds3-utils/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java similarity index 100% rename from ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java rename to ds3-utils/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 000000000..8bee35caa --- /dev/null +++ b/gradle.properties @@ -0,0 +1,11 @@ +commonscodecVersion=1.10 +commonsioVersion=2.4 +commonslangVersion=3.6 +guavaVersion=23.0 +hamcrestVersion=1.3 +httpclientVersion=4.5.3 +jacksonVersion=2.9.0 +jnaVersion=4.2.2 +junitVersion=4.12 +mockitoVersion=1.10.+ +slf4jVersion=1.7.25 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7b61d71aa..145412991 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 258c82709..c57087b3b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Mon Jun 05 12:51:36 MDT 2017 +#Fri Aug 25 15:48:59 MDT 2017 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip