From 1823a802bd5979b5090fb8580afe3154b7ff33d2 Mon Sep 17 00:00:00 2001 From: Eric <192110+scribe@users.noreply.github.com> Date: Mon, 15 Apr 2019 12:15:47 -0600 Subject: [PATCH 1/3] This still needs testing but this is the sparse fix --- .../ds3client/helpers/FileObjectGetter.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index 4f99d36ee..9adf01d19 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -33,6 +33,7 @@ public class FileObjectGetter implements ObjectChannelBuilder { /** * Creates a new FileObjectGetter to retrieve files from a remote DS3 system to the local file system. + * * @param root The {@code root} directory of the local file system for all files being transferred. */ public FileObjectGetter(final Path root) { @@ -47,14 +48,24 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { Files.createDirectories(FileUtils.resolveForSymbolic(parentPath)); } - if ( ! FileUtils.isTransferablePath(objectPath)) { + if (!FileUtils.isTransferablePath(objectPath)) { throw new UnrecoverableIOException(objectPath + " is not a regular file."); } - return FileChannel.open( - objectPath, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE - ); + synchronized (objectPath.toString().intern()) { + if (Files.notExists(objectPath)) { + Files.createDirectories(objectPath.getParent()); + return FileChannel.open( + objectPath, + StandardOpenOption.SPARSE, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE_NEW + ); + } + return FileChannel.open( + objectPath, + StandardOpenOption.WRITE + ); + } } -} +} \ No newline at end of file From c4b85cec796e03dd3d16ad920eeb9add19063c81 Mon Sep 17 00:00:00 2001 From: Eric <192110+scribe@users.noreply.github.com> Date: Thu, 18 Apr 2019 13:11:19 -0600 Subject: [PATCH 2/3] using slices --- .../ds3client/helpers/FileObjectGetter.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index 9adf01d19..ed5661073 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -15,6 +15,7 @@ package com.spectralogic.ds3client.helpers; +import com.google.common.util.concurrent.Striped; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; import com.spectralogic.ds3client.utils.FileUtils; @@ -24,12 +25,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.concurrent.locks.Lock; /** * Writes files to the local file system preserving the path. */ public class FileObjectGetter implements ObjectChannelBuilder { private final Path root; + private final Striped striped; /** * Creates a new FileObjectGetter to retrieve files from a remote DS3 system to the local file system. @@ -38,6 +41,7 @@ public class FileObjectGetter implements ObjectChannelBuilder { */ public FileObjectGetter(final Path root) { this.root = root; + this.striped = Striped.lazyWeakLock(10); } @Override @@ -52,7 +56,8 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { throw new UnrecoverableIOException(objectPath + " is not a regular file."); } - synchronized (objectPath.toString().intern()) { + try { + striped.get(key).lock(); if (Files.notExists(objectPath)) { Files.createDirectories(objectPath.getParent()); return FileChannel.open( @@ -62,10 +67,12 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { StandardOpenOption.CREATE_NEW ); } - return FileChannel.open( - objectPath, - StandardOpenOption.WRITE - ); + } finally { + striped.get(key).unlock(); } + return FileChannel.open( + objectPath, + StandardOpenOption.WRITE + ); } } \ No newline at end of file From 878c16a57e95b89b564f74e00e6d20a0c51889b6 Mon Sep 17 00:00:00 2001 From: Eric <192110+scribe@users.noreply.github.com> Date: Tue, 23 Apr 2019 10:08:38 -0600 Subject: [PATCH 3/3] removing extra get for slice --- .../com/spectralogic/ds3client/helpers/FileObjectGetter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java index ed5661073..d942e9327 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/FileObjectGetter.java @@ -56,8 +56,9 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { throw new UnrecoverableIOException(objectPath + " is not a regular file."); } + final Lock lock = striped.get(key); try { - striped.get(key).lock(); + lock.lock(); if (Files.notExists(objectPath)) { Files.createDirectories(objectPath.getParent()); return FileChannel.open( @@ -68,7 +69,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { ); } } finally { - striped.get(key).unlock(); + lock.unlock(); } return FileChannel.open( objectPath,