From 8446c2dca0805663c1879ce64041e0f15d77c9ea Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 6 Dec 2016 15:55:12 -0700 Subject: [PATCH 1/4] If writing to a file fails, we throw an exception that prevents GetObject retry logic from retrying. --- .../integration/GetJobManagement_Test.java | 195 ++++++++++++++++++ .../helpers/ExceptionClassifier.java | 5 + .../helpers/UnrecoverableIOException.java | 36 ++++ .../ds3client/utils/FileSystemUtils.java | 32 +++ .../spectralogic/ds3client/utils/IOUtils.java | 13 +- .../helpers/ExceptionClassifier_Test.java | 57 +++++ 6 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java create mode 100644 ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/ExceptionClassifier_Test.java diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java index b35547542..5bcdc6a49 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/GetJobManagement_Test.java @@ -48,11 +48,14 @@ import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.channels.WritableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.security.AccessControlException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -72,6 +75,7 @@ public class GetJobManagement_Test { private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client); private static final String BUCKET_NAME = "Get_Job_Management_Test"; private static final String TEST_ENV_NAME = "GetJobManagement_Test"; + private static final String DISK_FULL_MESSAGE = "There is not enough space on the disk"; private static TempStorageIds envStorageIds; private static UUID dataPolicyId; @@ -223,6 +227,196 @@ private void deleteBigFileFromBlackPearlBucket() throws IOException { } } + @Test(expected = RuntimeException.class) + public void testReadRetrybugFixWithUnwritableDirectory() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + putBigFile(); + + final String tempPathPrefix = null; + final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + final File tempDirectory = tempDirectoryPath.toFile(); + tempDirectory.setWritable(false); + + try { + final String DIR_NAME = "largeFiles/"; + final String FILE_NAME = "lesmis-copies.txt"; + + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize); + + final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl)client); + + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 3; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj)); + + final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim + .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); + + assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue())); + + readJob.transfer(new FileObjectGetter(tempDirectoryPath)); + + final File originalFile = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME).toFile(); + final File fileCopiedFromBP = Paths.get(tempDirectoryPath.toString(), FILE_NAME).toFile(); + assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP)); + + } finally { + tempDirectory.setReadable(true); + tempDirectory.setWritable(true); + tempDirectory.setExecutable(true); + FileUtils.deleteDirectory(tempDirectoryPath.toFile()); + deleteBigFileFromBlackPearlBucket(); + } + } + + @Test(expected = AccessControlException.class) + public void testReadRetrybugWhenChannelThrowsAccessException() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + putBigFile(); + + final String tempPathPrefix = null; + final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + try { + final String DIR_NAME = "largeFiles/"; + final String FILE_NAME = "lesmis-copies.txt"; + + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize); + + final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl)client); + + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 3; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj)); + + final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim + .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); + + assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue())); + + readJob.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() { + @Override + public SeekableByteChannel buildChannel(final String key) throws IOException { + throw new AccessControlException(key); + } + }); + } finally { + FileUtils.deleteDirectory(tempDirectoryPath.toFile()); + deleteBigFileFromBlackPearlBucket(); + } + } + + @Test + public void testReadRetryBugWhenDiskIsFull() throws IOException, URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { + putBigFile(); + + final String tempPathPrefix = null; + final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix); + + try { + final String DIR_NAME = "largeFiles/"; + final String FILE_NAME = "lesmis-copies.txt"; + + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME); + final long bookSize = Files.size(objPath); + final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize); + + final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl)client); + + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 3; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj)); + + final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim + .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId())); + + assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue())); + + try { + readJob.transfer(new FailingChannelBuilder()); + } catch (final UnrecoverableIOException e) { + assertEquals(DISK_FULL_MESSAGE, e.getCause().getMessage()); + } + } finally { + FileUtils.deleteDirectory(tempDirectoryPath.toFile()); + deleteBigFileFromBlackPearlBucket(); + } + } + + private static class FailingChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder { + @Override + public SeekableByteChannel buildChannel(final String key) throws IOException { + final String filePath = key; + final SeekableByteChannel seekableByteChannel = Files.newByteChannel(Paths.get(filePath), + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, + StandardOpenOption.DELETE_ON_CLOSE); + return new SeekableByteChannelWrapper(seekableByteChannel); + } + } + + private static class SeekableByteChannelWrapper implements SeekableByteChannel { + private final SeekableByteChannel seekableByteChannel; + + private SeekableByteChannelWrapper(final SeekableByteChannel seekableByteChannel) { + this.seekableByteChannel = seekableByteChannel; + } + + @Override + public int read(final ByteBuffer dst) throws IOException { + return seekableByteChannel.read(dst); + } + + @Override + public int write(final ByteBuffer src) throws IOException { + throw new IOException(DISK_FULL_MESSAGE); + } + + @Override + public long position() throws IOException { + return seekableByteChannel.position(); + } + + @Override + public SeekableByteChannel position(final long newPosition) throws IOException { + return seekableByteChannel.position(newPosition); + } + + @Override + public long size() throws IOException { + return seekableByteChannel.size(); + } + + @Override + public SeekableByteChannel truncate(final long size) throws IOException { + return seekableByteChannel.truncate(size); + } + + @Override + public boolean isOpen() { + return seekableByteChannel.isOpen(); + } + + @Override + public void close() throws IOException { + seekableByteChannel.close(); + } + } + + @Test public void createReadJobWithPriorityOption() throws IOException, InterruptedException, URISyntaxException { @@ -324,6 +518,7 @@ public void objectCompleted(final String name) { } } + @Test public void testFiringFailureHandlerWhenGettingChunks() throws URISyntaxException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, IOException diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java index 7d8e863e1..9ecfcc163 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java @@ -16,11 +16,16 @@ package com.spectralogic.ds3client.helpers; import java.io.IOException; +import java.nio.file.FileSystemException; public final class ExceptionClassifier { private ExceptionClassifier() { } public static boolean isRecoverableException(final Throwable t) { + if (t.getClass().equals(UnrecoverableIOException.class) || t instanceof FileSystemException || t instanceof SecurityException) { + return false; + } + return t instanceof IOException; } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java new file mode 100644 index 000000000..f1f69bc19 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java @@ -0,0 +1,36 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import java.io.IOException; + +public class UnrecoverableIOException extends IOException { + public UnrecoverableIOException() { + super(); + } + + public UnrecoverableIOException(final String message) { + super(message); + } + + public UnrecoverableIOException(final String message, final Throwable cause) { + super(message, cause); + } + + public UnrecoverableIOException(final Throwable cause) { + super(cause); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java new file mode 100644 index 000000000..4d82d441e --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java @@ -0,0 +1,32 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils; + +import com.google.common.base.Preconditions; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public final class FileSystemUtils { + private FileSystemUtils() {} + + public static long getAvailableFileSpace(final Path path) throws IOException { + Preconditions.checkNotNull(path, "path must not be null."); + + return Files.getFileStore(path).getUsableSpace(); + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/IOUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/IOUtils.java index b48d81e6d..45b00c63c 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/IOUtils.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/IOUtils.java @@ -15,6 +15,8 @@ package com.spectralogic.ds3client.utils; +import com.spectralogic.ds3client.helpers.UnrecoverableIOException; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -66,9 +68,14 @@ public static long copy( long statusUpdateTime = startTime; while ((len = inputStream.read(buffer)) != -1) { totalBytes += len; - byteBuffer.position(0); - byteBuffer.limit(len); - writableByteChannel.write(byteBuffer); + + try { + byteBuffer.position(0); + byteBuffer.limit(len); + writableByteChannel.write(byteBuffer); + } catch (final Throwable t) { + throw new UnrecoverableIOException(t); + } final long curTime = PerformanceUtils.getCurrentTime(); if (statusUpdateTime <= curTime) { diff --git a/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/ExceptionClassifier_Test.java b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/ExceptionClassifier_Test.java new file mode 100644 index 000000000..5d116f167 --- /dev/null +++ b/ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/ExceptionClassifier_Test.java @@ -0,0 +1,57 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.FileSystemException; +import java.security.AccessControlException; + +import static org.junit.Assert.assertTrue; + +public class ExceptionClassifier_Test { + @Test + public void testThatAccessDeniedExceptionIsUnrecoverable() { + assertTrue(ExceptionClassifier.isUnrecoverableException(new AccessDeniedException("a file name"))); + } + + @Test + public void testThatSecurityExceptionIsUnrecoverable() { + assertTrue(ExceptionClassifier.isUnrecoverableException(new SecurityException())); + } + + @Test + public void testThatFileSystemExceptionIsUnrecoverable() { + assertTrue(ExceptionClassifier.isUnrecoverableException(new FileSystemException("a file name"))); + } + + @Test + public void testThatAccessControlExceptionIsUnrecoverable() { + assertTrue(ExceptionClassifier.isUnrecoverableException(new AccessControlException("access control exception"))); + } + + @Test + public void testThatUnrecoverableIOExceptionIsUnrecoverable() { + assertTrue(ExceptionClassifier.isUnrecoverableException(new UnrecoverableIOException())); + } + + @Test + public void testThatIOExceptionIsRecoverable() { + assertTrue(ExceptionClassifier.isRecoverableException(new IOException())); + } +} From 43572f6b0a179ffd0c774a97de9c942b31019b26 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Tue, 6 Dec 2016 21:05:50 -0700 Subject: [PATCH 2/4] Allowing for subclassing UnrecoverableIOException. --- .../com/spectralogic/ds3client/helpers/ExceptionClassifier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java index 9ecfcc163..963740dd1 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ExceptionClassifier.java @@ -22,7 +22,7 @@ public final class ExceptionClassifier { private ExceptionClassifier() { } public static boolean isRecoverableException(final Throwable t) { - if (t.getClass().equals(UnrecoverableIOException.class) || t instanceof FileSystemException || t instanceof SecurityException) { + if (t instanceof UnrecoverableIOException || t instanceof FileSystemException || t instanceof SecurityException) { return false; } From c5b5550c5eae018da2fbffbe992983c3efd3e839 Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Wed, 7 Dec 2016 16:59:10 -0700 Subject: [PATCH 3/4] Adding a helper method to check for sufficient file space for customers to use instead of catching an exception when a read job fails. --- .../helpers/FileSystemHelper_Test.java | 331 ++++++++++++++++++ .../integration/PutJobManagement_Test.java | 3 - .../ds3client/helpers/Ds3ClientHelpers.java | 16 + .../helpers/Ds3ClientHelpersImpl.java | 41 ++- .../ds3client/helpers/FileSystemHelper.java | 49 +++ .../helpers/FileSystemHelperImpl.java | 145 ++++++++ .../ObjectStorageSpaceVerificationResult.java | 127 +++++++ .../ds3client/utils/FileSystemUtils.java | 32 -- 8 files changed, 708 insertions(+), 36 deletions(-) create mode 100644 ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelper.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java create mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectStorageSpaceVerificationResult.java delete mode 100644 ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java new file mode 100644 index 000000000..925d00605 --- /dev/null +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/helpers/FileSystemHelper_Test.java @@ -0,0 +1,331 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner; +import com.spectralogic.ds3client.integration.Util; +import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds; +import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil; +import com.spectralogic.ds3client.models.ChecksumType; +import com.spectralogic.ds3client.models.bulk.Ds3Object; +import com.spectralogic.ds3client.utils.ResourceUtils; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static com.spectralogic.ds3client.integration.Util.deleteAllContents; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; + +public class FileSystemHelper_Test { + private static final Ds3Client client = Util.fromEnv(); + private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client); + private static final String BUCKET_NAME = "File_System_Helper_Test"; + private static final String TEST_ENV_NAME = "FileSystem_Helper_Test"; + private static TempStorageIds envStorageIds; + private static UUID envDataPolicyId; + + @BeforeClass + public static void startup() throws IOException { + envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client); + envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, client); + } + + @Before + public void setupBucket() throws IOException { + HELPERS.ensureBucketExists(BUCKET_NAME, envDataPolicyId); + } + + @AfterClass + public static void teardown() throws IOException { + TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, client); + client.close(); + } + + @Test + public void testObjectsFitBucketThatHasContent() throws IOException, URISyntaxException { + try { + final String DIR_NAME = "largeFiles/"; + final String[] FILE_NAMES = new String[]{"lesmis-copies.txt"}; + + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final AtomicLong totalBookSizes = new AtomicLong(0); + + final List bookTitles = new ArrayList<>(); + final List objects = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + totalBookSizes.getAndAdd(bookSize); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objects.add(obj); + } + + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 1; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final AtomicInteger numTimesCallbackCalled = new AtomicInteger(0); + + final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects); + writeJob.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + numTimesCallbackCalled.getAndIncrement(); + + final ObjectStorageSpaceVerificationResult objectStorageSpaceVerificationResult = + ds3ClientHelpers.objectsFromBucketWillFitInDirectory(BUCKET_NAME, + Arrays.asList(FILE_NAMES), + Paths.get(".")); + + assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.OK, objectStorageSpaceVerificationResult.getVerificationStatus()); + assertEquals(objectStorageSpaceVerificationResult.getRequiredSpace(), totalBookSizes.get()); + assertTrue(objectStorageSpaceVerificationResult.getAvailableSpace() > 0); + assertTrue(objectStorageSpaceVerificationResult.containsSufficientSpace()); + assertNull(objectStorageSpaceVerificationResult.getIoException()); + } + }); + + writeJob.transfer(new FileObjectPutter(dirPath)); + + assertEquals(1, numTimesCallbackCalled.get()); + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + @Test + public void testObjectsFitBucketWithNonExistentBucket() { + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 1; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory( + "bad bucket name", Arrays.asList(new String[] {}), Paths.get(".")); + + assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.BucketDoesNotExist, result.getVerificationStatus()); + assertEquals(0, result.getRequiredSpace()); + assertEquals(0, result.getAvailableSpace()); + assertFalse(result.containsSufficientSpace()); + assertNull(result.getIoException()); + } + + @Test + public void testObjectsFitBucketWithPathNotDirectory() throws IOException { + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 1; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final Path textFile = Files.createFile(Paths.get("Gracie.txt")); + + try { + final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory( + "bad bucket name", Arrays.asList(new String[]{}), textFile); + + assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathIsNotADirectory, result.getVerificationStatus()); + assertEquals(0, result.getRequiredSpace()); + assertEquals(0, result.getAvailableSpace()); + assertFalse(result.containsSufficientSpace()); + assertNull(result.getIoException()); + } finally { + Files.delete(textFile); + } + } + + @Test + public void testObjectsFitBucketPathDoesNotExist() throws IOException { + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 1; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final Path directory = Files.createDirectory(Paths.get("dir")); + FileUtils.deleteDirectory(directory.toFile()); + + final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory( + "bad bucket name", Arrays.asList(new String[]{}), directory); + + assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathDoesNotExist, result.getVerificationStatus()); + assertEquals(0, result.getRequiredSpace()); + assertEquals(0, result.getAvailableSpace()); + assertFalse(result.containsSufficientSpace()); + assertNull(result.getIoException()); + } + + @Test + public void testObjectsFitBucketPathLacksAccess() throws IOException { + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 1; + final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts); + + final Path directory = Files.createDirectory(Paths.get("dir")); + directory.toFile().setWritable(false); + + try { + final ObjectStorageSpaceVerificationResult result = ds3ClientHelpers.objectsFromBucketWillFitInDirectory( + "bad bucket name", Arrays.asList(new String[]{}), directory); + + assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksAccess, result.getVerificationStatus()); + assertEquals(0, result.getRequiredSpace()); + assertEquals(0, result.getAvailableSpace()); + assertFalse(result.containsSufficientSpace()); + assertNull(result.getIoException()); + } finally { + directory.toFile().setWritable(true); + FileUtils.deleteDirectory(directory.toFile()); + } + } + + @Test + public void testObjectsFitBucketPathLacksSpace() throws IOException, URISyntaxException { + putObjectThenRunVerification(new MockedFileSystemHelper(), new ResultVerifier() { + @Override + public void verifyResult(final ObjectStorageSpaceVerificationResult result, + final long totalRequiredSize) + { + assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksSufficientStorageSpace, result.getVerificationStatus()); + assertEquals(totalRequiredSize, result.getRequiredSpace()); + assertEquals(-1, result.getAvailableSpace()); + assertFalse(result.containsSufficientSpace()); + assertNull(result.getIoException()); + } + }); + } + + private interface ResultVerifier { + void verifyResult(final ObjectStorageSpaceVerificationResult result, final long totalRequiredSize); + } + + private void putObjectThenRunVerification(final FileSystemHelper fileSystemHelper, + final ResultVerifier resultVerifier) + throws IOException, URISyntaxException + { + try { + final String DIR_NAME = "largeFiles/"; + final String[] FILE_NAMES = new String[]{"lesmis-copies.txt"}; + + final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME); + + final AtomicLong totalBookSizes = new AtomicLong(0); + + final List bookTitles = new ArrayList<>(); + final List objects = new ArrayList<>(); + for (final String book : FILE_NAMES) { + final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book); + final long bookSize = Files.size(objPath); + totalBookSizes.getAndAdd(bookSize); + final Ds3Object obj = new Ds3Object(book, bookSize); + + bookTitles.add(book); + objects.add(obj); + } + + final int maxNumBlockAllocationRetries = 1; + final int maxNumObjectTransferAttempts = 1; + final int retryDelay = -1; + final Ds3ClientHelpers ds3ClientHelpers = new Ds3ClientHelpersImpl(client, + maxNumBlockAllocationRetries, + maxNumObjectTransferAttempts, + retryDelay, + new SameThreadEventRunner(), + fileSystemHelper); + + final AtomicInteger numTimesCallbackCalled = new AtomicInteger(0); + + final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects); + writeJob.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + numTimesCallbackCalled.getAndIncrement(); + + final ObjectStorageSpaceVerificationResult result = + ds3ClientHelpers.objectsFromBucketWillFitInDirectory(BUCKET_NAME, + Arrays.asList(FILE_NAMES), + Paths.get(".")); + + resultVerifier.verifyResult(result, totalBookSizes.get()); + } + }); + + writeJob.transfer(new FileObjectPutter(dirPath)); + + assertEquals(1, numTimesCallbackCalled.get()); + } finally { + deleteAllContents(client, BUCKET_NAME); + } + } + + private static class MockedFileSystemHelper extends FileSystemHelperImpl { + @Override + public long getAvailableFileSpace(final Path path) throws IOException { + return -1L; + } + } + + @Test + public void testObjectsFitBucketPathThrows() throws IOException, URISyntaxException { + putObjectThenRunVerification(new MockedFileSystemHelperThrows(), + new ResultVerifier() { + @Override + public void verifyResult(final ObjectStorageSpaceVerificationResult result, + final long totalRequiredSize) + { + assertEquals(ObjectStorageSpaceVerificationResult.VerificationStatus.CaughtIOException, result.getVerificationStatus()); + assertEquals(totalRequiredSize, result.getRequiredSpace()); + assertEquals(0, result.getAvailableSpace()); + assertFalse(result.containsSufficientSpace()); + assertNotNull(result.getIoException()); + } + }); + } + + private static class MockedFileSystemHelperThrows extends FileSystemHelperImpl { + @Override + public long getAvailableFileSpace(final Path path) throws IOException { + throw new IOException("IOExceptionAtor"); + } + } +} diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java index bd54754a7..b144a40f0 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/PutJobManagement_Test.java @@ -40,9 +40,6 @@ import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel; import com.spectralogic.ds3client.utils.ResourceUtils; -import com.spectralogic.ds3client.IntValue; - -import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimWithFailedChunkAllocation; import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory.ClientFailureType; import org.apache.commons.io.FileUtils; diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java index 354edfd61..95c327eee 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.nio.channels.SeekableByteChannel; import java.nio.file.Path; +import java.util.Collection; import java.util.Map; import java.util.UUID; @@ -382,4 +383,19 @@ public static String stripLeadingPath(final String objectName, final String pref } return returnString; } + + /** + * Determine if the file system directory specified in the destinationDirectory parameter + * has enough storage space to contain the objects listed in the parameter objectNames contained in + * the bucket specified in the parameter buckName. You can use this method prior to starting a read + * job to ensure that your file system has enough storage space to contain the objects you wish to + * retrieve. + * @param bucketName The Black Pearl bucket containing the objects you wish to retrieve. + * @param objectNames The names of the objects you wish to retrieve. + * @param destinationDirectory The file system directory in you intend to store retrieved objects. + * @return {@link ObjectStorageSpaceVerificationResult} + */ + public abstract ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(final String bucketName, + final Collection objectNames, + final Path destinationDirectory); } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java index 3e771fcd3..da6d6418f 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java @@ -45,6 +45,7 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collection; import java.util.List; import java.util.UUID; @@ -62,6 +63,7 @@ class Ds3ClientHelpersImpl extends Ds3ClientHelpers { private final int retryDelay; private final int objectTransferAttempts; private final EventRunner eventRunner; + private final FileSystemHelper fileSystemHelper; public Ds3ClientHelpersImpl(final Ds3Client client) { this(client, DEFAULT_RETRY_AFTER); @@ -79,12 +81,28 @@ public Ds3ClientHelpersImpl(final Ds3Client client, final int retryAfter, final this(client, retryAfter, objectTransferAttempts, retryDelay, new SameThreadEventRunner()); } - public Ds3ClientHelpersImpl(final Ds3Client client, final int retryAfter, final int objectTransferAttempts, final int retryDelay, final EventRunner eventRunner) { + public Ds3ClientHelpersImpl(final Ds3Client client, + final int retryAfter, + final int objectTransferAttempts, + final int retryDelay, + final EventRunner eventRunner) + { + this(client, retryAfter, objectTransferAttempts, retryDelay, eventRunner, new FileSystemHelperImpl()); + } + + public Ds3ClientHelpersImpl(final Ds3Client client, + final int retryAfter, + final int objectTransferAttempts, + final int retryDelay, + final EventRunner eventRunner, + final FileSystemHelper fileSystemHelper) + { this.client = client; this.retryAfter = retryAfter; this.objectTransferAttempts = objectTransferAttempts; this.retryDelay = retryDelay; this.eventRunner = eventRunner; + this.fileSystemHelper = fileSystemHelper; } @Override @@ -321,4 +339,25 @@ public Ds3Object apply(@Nullable final Ds3Object object) { } }); } + + /** + * Determine if the file system directory specified in the destinationDirectory parameter + * has enough storage space to contain the objects listed in the parameter objectNames contained in + * the bucket specified in the parameter buckName. You can use this method prior to starting a read + * job to ensure that your file system has enough storage space to contain the objects you wish to + * retrieve. + * + * @param bucketName The Black Pearl bucket containing the objects you wish to retrieve. + * @param objectNames The names of the objects you wish to retrieve. + * @param destinationDirectory The file system directory in you intend to store retrieved objects. + * @return {@link ObjectStorageSpaceVerificationResult} + */ + @Override + public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(final String bucketName, + final Collection objectNames, + final Path destinationDirectory) + { + return fileSystemHelper.objectsFromBucketWillFitInDirectory(this, + bucketName, objectNames, destinationDirectory); + } } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelper.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelper.java new file mode 100644 index 000000000..cad6f85cf --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelper.java @@ -0,0 +1,49 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; + +public interface FileSystemHelper { + boolean pathIsDirectory(final Path path); + + boolean pathObjectExists(final Path path); + + boolean pathIsWritable(final Path path); + + long getAvailableFileSpace(final Path path) throws IOException; + + /** + * Determine if the file system directory specified in the destinationDirectory parameter + * has enough storage space to contain the objects listed in the parameter objectNames contained in + * the bucket specified in the parameter buckName. You can use this method prior to starting a read + * job to ensure that your file system has enough storage space to contain the objects you wish to + * retrieve. + * + * @param helpers An instance of the Ds3ClientHelpers interface that has most likely + * wrapped an instance of Ds3Client. + * @param bucketName The Black Pearl bucket containing the objects you wish to retrieve. + * @param objectNames The names of the objects you wish to retrieve. + * @param destinationDirectory The file system directory in you intend to store retrieved objects. + * @return {@link ObjectStorageSpaceVerificationResult} + */ + ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(final Ds3ClientHelpers helpers, + final String bucketName, + final Collection objectNames, + final Path destinationDirectory); +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java new file mode 100644 index 000000000..edc551955 --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java @@ -0,0 +1,145 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.google.common.base.Preconditions; +import com.spectralogic.ds3client.models.Contents; +import com.spectralogic.ds3client.utils.Guard; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +class FileSystemHelperImpl implements FileSystemHelper { + public boolean pathIsDirectory(final Path path) { + Preconditions.checkNotNull(path, "path must not be null."); + + return Files.isDirectory(path); + } + + public boolean pathObjectExists(final Path path) { + Preconditions.checkNotNull(path, "path must not be null."); + + return Files.exists(path); + } + + public boolean pathIsWritable(final Path path) { + Preconditions.checkNotNull(path, "path must not be null."); + + return Files.isWritable(path); + } + + public long getAvailableFileSpace(final Path path) throws IOException { + Preconditions.checkNotNull(path, "path must not be null."); + + return Files.getFileStore(path).getUsableSpace(); + } + + /** + * Determine if the file system directory specified in the destinationDirectory parameter + * has enough storage space to contain the objects listed in the parameter objectNames contained in + * the bucket specified in the parameter buckName. You can use this method prior to starting a read + * job to ensure that your file system has enough storage space to contain the objects you wish to + * retrieve. + * + * @param helpers An instance of the Ds3ClientHelpers interface that has most likely + * wrapped an instance of Ds3Client. + * @param bucketName The Black Pearl bucket containing the objects you wish to retrieve. + * @param objectNames The names of the objects you wish to retrieve. + * @param destinationDirectory The file system directory in you intend to store retrieved objects. + * @return {@link ObjectStorageSpaceVerificationResult} + */ + public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory(final Ds3ClientHelpers helpers, + final String bucketName, + final Collection objectNames, + final Path destinationDirectory) + { + Preconditions.checkNotNull(helpers, "helpers may not be null."); + Guard.throwOnNullOrEmptyString(bucketName, "bucketName must have a non-empty value."); + Preconditions.checkNotNull(objectNames, "objectNames may not be null."); + Preconditions.checkNotNull(destinationDirectory, "destinationDirectory may not be null."); + + long requiredSpace = 0; + long availableSpace = 0; + final IOException ioException = null; + + if ( ! pathObjectExists(destinationDirectory)) { + return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.PathDoesNotExist, + requiredSpace, availableSpace, ioException); + } + + if ( ! pathIsDirectory(destinationDirectory)) { + return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.PathIsNotADirectory, + requiredSpace, availableSpace, ioException); + } + + if ( ! pathIsWritable(destinationDirectory)) { + return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksAccess, + requiredSpace, availableSpace, ioException); + } + + try { + helpers.ensureBucketExists(bucketName); + } catch (final IOException e) { + return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.BucketDoesNotExist, + requiredSpace, availableSpace, ioException); + } + + try { + requiredSpace = getRequiredSpaceForObjects(getObjectSizes(helpers, bucketName, objectNames)); + availableSpace = getAvailableFileSpace(destinationDirectory); + } catch (final IOException e) { + return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.CaughtIOException, + requiredSpace, availableSpace, e); + } + + final ObjectStorageSpaceVerificationResult.VerificationStatus verificationStatus = availableSpace > requiredSpace ? + ObjectStorageSpaceVerificationResult.VerificationStatus.OK : ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksSufficientStorageSpace; + + return new ObjectStorageSpaceVerificationResult(verificationStatus, requiredSpace, availableSpace, ioException); + } + + private Map getObjectSizes(final Ds3ClientHelpers helpers, + final String bucketName, + final Collection objectNames) + throws IOException + { + final Map objectSizeMap = new HashMap<>(); + + final Iterable bucketContents = helpers.listObjects(bucketName); + + for (final Contents bucketContent : bucketContents) { + objectSizeMap.put(bucketContent.getKey(), bucketContent.getSize()); + } + + objectSizeMap.keySet().retainAll(objectNames); + + return objectSizeMap; + } + + private long getRequiredSpaceForObjects(final Map objectSizeMap) { + long result = 0; + + for (final long objectSize : objectSizeMap.values()) { + result += objectSize; + } + + return result; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectStorageSpaceVerificationResult.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectStorageSpaceVerificationResult.java new file mode 100644 index 000000000..e690752ca --- /dev/null +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectStorageSpaceVerificationResult.java @@ -0,0 +1,127 @@ +/* + * **************************************************************************** + * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; + +/** + * An object that holds the result of calling {@link Ds3ClientHelpers#objectsFromBucketWillFitInDirectory(String, Collection, Path)} + */ +public class ObjectStorageSpaceVerificationResult { + public enum VerificationStatus { + /** + * Objects will fit in intended directory. + */ + OK, + + /** + * The path intended to store objects is not a directory. + */ + PathIsNotADirectory, + + /** + * The path intended to store objects does not exist, so its storage capacity is unknown. + */ + PathDoesNotExist, + + /** + * The path intended to store objects is not writable. + */ + PathLacksAccess, + + /** + * The path intended to store objects lacks the capacity to store all the objects. + */ + PathLacksSufficientStorageSpace, + + /** + * The bucket from which you wish to retrieve objects does not exist. + */ + BucketDoesNotExist, + + /** + * Caught an IOException from the file system or communicating with Black Pearl + */ + CaughtIOException + } + + private final VerificationStatus verificationStatus; + private final long requiredSpace; + private final long availableSpace; + private final boolean containsSufficientSpace; + private final IOException ioException; + + public ObjectStorageSpaceVerificationResult(final VerificationStatus verificationStatus, + final long requiredSpace, + final long availableSpace, + final IOException ioException) + { + this.requiredSpace = requiredSpace; + this.availableSpace = availableSpace; + this.containsSufficientSpace = (availableSpace > requiredSpace); + this.verificationStatus = verificationStatus; + this.ioException = ioException; + } + + /** + * @return The status object describing the ability of a path to contain a collection of objects. + */ + public VerificationStatus getVerificationStatus() { + return verificationStatus; + } + + /** + * @return The space required, in bytes, to store the requested objects. + */ + public long getRequiredSpace() { + return requiredSpace; + } + + /** + * @return The space available, in bytes, available in a destination path. + */ + public long getAvailableSpace() { + return availableSpace; + } + + /** + * When calling {@link Ds3ClientHelpers#objectsFromBucketWillFitInDirectory(String, Collection, Path)}, you can + * quickly determine the result using code like the following... + * + * 
+     *     
+     *         if (helpers.objectsFromBucketWillFitInDirectory(bucketName, objectsToRetrieve, directory)).containsSufficientSpace() {
+     *             // put read job here
+     *         }
+     *     
+     *
+ * @return + */ + public boolean containsSufficientSpace() { + return containsSufficientSpace; + } + + /** + * @return Any IOException generated as a result of calling + * {@link Ds3ClientHelpers#objectsFromBucketWillFitInDirectory(String, Collection, Path)}. Null if no + * IOException was generated. + */ + public IOException getIoException() { + return ioException; + } +} diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java deleted file mode 100644 index 4d82d441e..000000000 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/utils/FileSystemUtils.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * **************************************************************************** - * Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.utils; - -import com.google.common.base.Preconditions; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -public final class FileSystemUtils { - private FileSystemUtils() {} - - public static long getAvailableFileSpace(final Path path) throws IOException { - Preconditions.checkNotNull(path, "path must not be null."); - - return Files.getFileStore(path).getUsableSpace(); - } -} From 47c0c381b3a26736621a827660c872cf3e7558fb Mon Sep 17 00:00:00 2001 From: GraciesPadre Date: Mon, 12 Dec 2016 13:56:08 -0700 Subject: [PATCH 4/4] Addressing code review comments --- .../helpers/FileSystemHelperImpl.java | 25 ++++++++++--------- .../helpers/UnrecoverableIOException.java | 8 ++++++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java index edc551955..690633a86 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileSystemHelperImpl.java @@ -77,19 +77,21 @@ public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory( long requiredSpace = 0; long availableSpace = 0; - final IOException ioException = null; if ( ! pathObjectExists(destinationDirectory)) { + final IOException ioException = null; return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.PathDoesNotExist, requiredSpace, availableSpace, ioException); } if ( ! pathIsDirectory(destinationDirectory)) { + final IOException ioException = null; return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.PathIsNotADirectory, requiredSpace, availableSpace, ioException); } if ( ! pathIsWritable(destinationDirectory)) { + final IOException ioException = null; return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksAccess, requiredSpace, availableSpace, ioException); } @@ -97,12 +99,13 @@ public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory( try { helpers.ensureBucketExists(bucketName); } catch (final IOException e) { + final IOException ioException = null; return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.BucketDoesNotExist, requiredSpace, availableSpace, ioException); } try { - requiredSpace = getRequiredSpaceForObjects(getObjectSizes(helpers, bucketName, objectNames)); + requiredSpace = getRequiredSpaceForObjects(helpers, bucketName, objectNames); availableSpace = getAvailableFileSpace(destinationDirectory); } catch (final IOException e) { return new ObjectStorageSpaceVerificationResult(ObjectStorageSpaceVerificationResult.VerificationStatus.CaughtIOException, @@ -112,14 +115,17 @@ public ObjectStorageSpaceVerificationResult objectsFromBucketWillFitInDirectory( final ObjectStorageSpaceVerificationResult.VerificationStatus verificationStatus = availableSpace > requiredSpace ? ObjectStorageSpaceVerificationResult.VerificationStatus.OK : ObjectStorageSpaceVerificationResult.VerificationStatus.PathLacksSufficientStorageSpace; + final IOException ioException = null; return new ObjectStorageSpaceVerificationResult(verificationStatus, requiredSpace, availableSpace, ioException); } - private Map getObjectSizes(final Ds3ClientHelpers helpers, - final String bucketName, - final Collection objectNames) - throws IOException + private long getRequiredSpaceForObjects(final Ds3ClientHelpers helpers, + final String bucketName, + final Collection objectNames) + throws IOException { + long result = 0; + final Map objectSizeMap = new HashMap<>(); final Iterable bucketContents = helpers.listObjects(bucketName); @@ -128,14 +134,9 @@ private Map getObjectSizes(final Ds3ClientHelpers helpers, objectSizeMap.put(bucketContent.getKey(), bucketContent.getSize()); } + // Of the objects in the bucket, keep the information about only those in objectNames objectSizeMap.keySet().retainAll(objectNames); - return objectSizeMap; - } - - private long getRequiredSpaceForObjects(final Map objectSizeMap) { - long result = 0; - for (final long objectSize : objectSizeMap.values()) { result += objectSize; } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java index f1f69bc19..0b2c2c834 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/UnrecoverableIOException.java @@ -17,6 +17,14 @@ import java.io.IOException; +/** + * An UnrecoverableIOException is used to classify IOExceptions into one of 2 types: + * those that should result in retrying a data transfer, such as a failure in an HTTP GET or PUT; + * and those that should not result in retrying a data transfer, such as a failure writing to + * or reading from a file. During an HTTP PUT or GET that involves transferring data to or from + * a file, file system failures are considered non-recoverable, where network-related failures + * are considered recoverable. + */ public class UnrecoverableIOException extends IOException { public UnrecoverableIOException() { super();