diff --git a/ds3-interfaces/build.gradle b/ds3-interfaces/build.gradle index 459717170..ac8ff43ef 100644 --- a/ds3-interfaces/build.gradle +++ b/ds3-interfaces/build.gradle @@ -22,7 +22,12 @@ buildscript { apply plugin: 'com.github.johnrengelman.shadow' +dependencies { + compile 'commons-io:commons-io:2.4' +} + shadowJar { + relocate 'org.apache', 'ds3fatjar.org.apache' dependencies { exclude(dependency('org.hamcrest:hamcrest-library:1.3')) exclude(dependency('org.mockito:mockito-core:1.10.19')) diff --git a/ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java b/ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java new file mode 100644 index 000000000..a351b3af7 --- /dev/null +++ b/ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java @@ -0,0 +1,60 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.exceptions; + +import org.apache.commons.io.output.StringBuilderWriter; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class AggregateException extends RuntimeException { + + private final List exceptions; + + public AggregateException(final Iterable exceptions) { + super("One or more exceptions were aggregated together"); + this.exceptions = exceptionList(exceptions); + } + + private static List exceptionList(final Iterable exceptions) { + final List exceptionList = new ArrayList<>(); + for (final Throwable t : exceptions) { + exceptionList.add(t); + } + return exceptionList; + } + + public Iterable getExceptions() { + return exceptions; + } + + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("One or more exceptions were aggregated:"); + + try (final StringBuilderWriter writer = new StringBuilderWriter(builder); + final PrintWriter printWriter = new PrintWriter(writer)) { + + for (final Throwable t : exceptions) { + printWriter.append("Exception: ").append(t.getMessage()); + t.printStackTrace(printWriter); + } + } + + return builder.toString(); + } +} diff --git a/ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers/MetadataAccess.java b/ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/MetadataAccess.java similarity index 93% rename from ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers/MetadataAccess.java rename to ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/MetadataAccess.java index da661256b..f9a0e5258 100644 --- a/ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers/MetadataAccess.java +++ b/ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/MetadataAccess.java @@ -13,7 +13,7 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.helpers.Ds3ClientHelpers; +package com.spectralogic.ds3client.helpers; import java.util.Map; diff --git a/ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java b/ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java new file mode 100644 index 000000000..857372dec --- /dev/null +++ b/ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java @@ -0,0 +1,73 @@ +/* + * **************************************************************************** + * Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.exceptions; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class AggregateException_Test { + + @Test + public void basicAggregate() { + final Exception e1 = new Exception("first exception"); + final Exception e2 = new Exception("second exception"); + + final List exceptionList = new ArrayList<>(); + exceptionList.add(e1); + exceptionList.add(e2); + + final AggregateException aggregateException = new AggregateException(exceptionList); + + final Iterator exceptionIter = aggregateException.getExceptions().iterator(); + + assertTrue(exceptionIter.hasNext()); + assertThat(exceptionIter.next(), is(notNullValue())); + assertTrue(exceptionIter.hasNext()); + assertThat(exceptionIter.next(), is(notNullValue())); + assertFalse(exceptionIter.hasNext()); + } + + @Test + public void repeatIteration() { + final Exception e1 = new Exception("first exception"); + + final List exceptionList = new ArrayList<>(); + exceptionList.add(e1); + + final AggregateException aggregateException = new AggregateException(exceptionList); + + final Iterator exceptionIter = aggregateException.getExceptions().iterator(); + + assertTrue(exceptionIter.hasNext()); + assertThat(exceptionIter.next(), is(notNullValue())); + assertFalse(exceptionIter.hasNext()); + + final Iterator repeatIter = aggregateException.getExceptions().iterator(); + assertTrue(repeatIter.hasNext()); + assertThat(repeatIter.next(), is(notNullValue())); + assertFalse(repeatIter.hasNext()); + + } +} diff --git a/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataAccessImpl.java b/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataAccessImpl.java index 01c45bc28..83e4c4f31 100644 --- a/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataAccessImpl.java +++ b/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataAccessImpl.java @@ -17,7 +17,7 @@ import com.google.common.collect.ImmutableMap; -import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.MetadataAccess; +import com.spectralogic.ds3client.helpers.MetadataAccess; import com.spectralogic.ds3client.metadata.interfaces.MetadataStore; import com.spectralogic.ds3client.utils.Platform; diff --git a/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataReceivedListenerImpl.java b/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataReceivedListenerImpl.java index ae7ba6507..90dd163c5 100644 --- a/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataReceivedListenerImpl.java +++ b/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataReceivedListenerImpl.java @@ -15,15 +15,24 @@ package com.spectralogic.ds3client.metadata; +import com.google.common.collect.ImmutableList; +import com.spectralogic.ds3client.exceptions.AggregateException; import com.spectralogic.ds3client.helpers.MetadataReceivedListener; import com.spectralogic.ds3client.metadata.interfaces.MetadataRestore; import com.spectralogic.ds3client.networking.Metadata; +import com.spectralogic.ds3client.utils.Guard; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; public class MetadataReceivedListenerImpl implements MetadataReceivedListener { + + private final static Logger LOG = LoggerFactory.getLogger(MetadataReceivedListenerImpl.class); + private final String localFilePath; + public MetadataReceivedListenerImpl(final String localFilePath) { this.localFilePath = localFilePath; } @@ -45,17 +54,41 @@ public void metadataReceived(final String filename, final Metadata metadata) { * @param metadata metadata which needs to be set on local file */ private void restoreMetaData(final String objectName, final Metadata metadata) throws IOException, InterruptedException { + + final ImmutableList.Builder exceptionBuilder = ImmutableList.builder(); + //get metadatarestore on the basis of os final MetadataRestore metadataRestore = new MetadataRestoreFactory().getOSSpecificMetadataRestore(metadata, objectName); //restore os name metadataRestore.restoreOSName(); + //restore user and owner based on OS - metadataRestore.restoreUserAndOwner(); + try { + metadataRestore.restoreUserAndOwner(); + } catch (final Throwable t) { + LOG.error("Could not restore owner and owner information", t); + exceptionBuilder.add(t); + } + //restore creation and modified time based on OS - metadataRestore.restoreFileTimes(); - //restore permissions based on OS - metadataRestore.restorePermissions(); - } + try { + metadataRestore.restoreFileTimes(); + } catch (final Throwable t) { + LOG.error("Could not restore the file times", t); + exceptionBuilder.add(t); + } + //restore permissions based on OS + try { + metadataRestore.restorePermissions(); + } catch (final Throwable t) { + LOG.error("Could not restore the file permissions", t); + exceptionBuilder.add(t); + } + final ImmutableList exceptions = exceptionBuilder.build(); + if (!Guard.isNullOrEmpty(exceptions)) { + throw new AggregateException(exceptions); + } + } } diff --git a/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore.java b/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore.java index 64faf8fa7..2add5293d 100644 --- a/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore.java +++ b/ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore.java @@ -18,7 +18,11 @@ import com.google.common.collect.ImmutableList; import com.spectralogic.ds3client.metadata.jna.Advapi32; import com.spectralogic.ds3client.networking.Metadata; +import com.spectralogic.ds3client.utils.Guard; import com.sun.jna.platform.win32.WinNT; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.IOException; import java.nio.file.FileSystems; @@ -36,6 +40,9 @@ import static com.spectralogic.ds3client.metadata.MetadataKeyConstants.KEY_OWNER; class WindowsMetadataRestore extends AbstractMetadataRestore { + + private static final Logger LOG = LoggerFactory.getLogger(WindowsMetadataRestore.class); + WindowsMetadataRestore(final Metadata metadata, final String filePath, final String localOS) { this.metadata = metadata; this.objectName = filePath; @@ -44,26 +51,31 @@ class WindowsMetadataRestore extends AbstractMetadataRestore { @Override public void restoreUserAndOwner() throws IOException { - if (storedOS.equals(localOS)) { - String ownerSid = null; - if (metadata.get(KEY_OWNER).size() > 0) { - ownerSid = metadata.get(KEY_OWNER).get(0); - } - String groupSid = null; - if (metadata.get(KEY_GROUP).size() > 0) { - groupSid = metadata.get(KEY_GROUP).get(0); - } - if (ownerSid != null && groupSid != null && !ownerSid.equals("") && !groupSid.equals("")) { - setOwnerIdandGroupId(ownerSid, groupSid); + if (storedOS != null && storedOS.equals(localOS)) { + + final String ownerSid = getMetadataProperty(metadata, KEY_OWNER); + final String groupSid = getMetadataProperty(metadata, KEY_GROUP); + + if (!Guard.isStringNullOrEmpty(ownerSid) && !Guard.isStringNullOrEmpty(groupSid)) { + setOwnerIdAndGroupId(ownerSid, groupSid); + } else { + LOG.warn("Cannot determine owner or group settings for {}", this.objectName); } + } else { + LOG.warn("The OS settings for owner and group properties cannot be restored for {}", this.objectName); } + } + private static String getMetadataProperty(final Metadata metadata, final String metadataName) { + return metadata.get(metadataName).get(0); } @Override public void restorePermissions() throws IOException, InterruptedException { if (storedOS != null && storedOS.equals(localOS)) { setPermissionsForWindows(); + } else { + LOG.warn("The OS settings for the file permissions cannot be restored for {}", this.objectName); } restoreFlags(); } @@ -76,19 +88,26 @@ private void setPermissionsForWindows() throws IOException { final String userListDisplay; final String[] users; final String[] usersDisplay; - if (metadata.get("ds3-userList").size() > 0) { + if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-userList"))) { userList = metadata.get("ds3-userList").get(0); - if (metadata.get("ds3-userListDisplay").size() > 0) { + if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-userListDisplay"))) { userListDisplay = metadata.get("ds3-userListDisplay").get(0); users = userList.split("-"); usersDisplay = userListDisplay.split("-"); for (int i = 0; i < users.length; i++) { - if (metadata.get("ds3-" + users[i]).size() > 0) { - final String ownerPermission = metadata.get("ds3-" + users[i]).get(0); + final String user = users[i]; + if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-" + user))) { + final String ownerPermission = metadata.get("ds3-" + user).get(0); restorePermissionByUser(ownerPermission, usersDisplay[i], aclEntryBuilder); + } else { + LOG.warn("Did not find any permissions for {} for file {}", user, this.objectName); } } + } else { + LOG.warn("There was not a 'userListDisplay' metadata entry for file {}, so we will not restore any permissions", this.objectName); } + } else { + LOG.warn("There was not a 'userList' metadata entry for file {}, so we will not restore any permissions", this.objectName); } aclAttributeView.setAcl(aclEntryBuilder.build()); @@ -148,7 +167,7 @@ private void restorePermissionByUser(final String permission, * @param ownerSidId sid of the owner * @param groupSidId sid of the group */ - private void setOwnerIdandGroupId(final String ownerSidId, final String groupSidId) throws IOException { + private void setOwnerIdAndGroupId(final String ownerSidId, final String groupSidId) throws IOException { final int infoType = WinNT.OWNER_SECURITY_INFORMATION | WinNT.GROUP_SECURITY_INFORMATION; final WinNT.PSIDByReference referenceOwner = new WinNT.PSIDByReference(); Advapi32.INSTANCE.ConvertStringSidToSid(ownerSidId, referenceOwner); @@ -166,7 +185,11 @@ private void restoreFlags() throws IOException, InterruptedException { if (metadata.get(KEY_FLAGS).size() > 0) { final String flags = metadata.get(KEY_FLAGS).get(0); restoreFlagsWindows(flags); + } else { + LOG.warn("The file flag settings do not exist for file {} and will not be restored", this.objectName); } + } else { + LOG.warn("The OS settings for restoring the file flags cannot be done for {}", this.objectName); } } @@ -199,9 +222,12 @@ private void restoreFlagsWindows(final String flag) throws IOException, Interrup stringBuilder.append(" -I"); stringBuilder.append(" -H"); } - stringBuilder.append(" " + "\"" + objectName + "\""); + stringBuilder.append(" " + "\"").append(objectName).append("\""); final Process p = Runtime.getRuntime().exec(stringBuilder.toString().split(" ")); - p.waitFor(); + final int returnCode = p.waitFor(); + if (returnCode != 0) { + LOG.error("Restoring the flag settings for file {} was not successful", this.objectName); + } } } diff --git a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java index 213d24204..c978e5038 100644 --- a/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java +++ b/ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java @@ -22,10 +22,7 @@ import com.spectralogic.ds3client.commands.*; import com.spectralogic.ds3client.commands.interfaces.BulkResponse; import com.spectralogic.ds3client.commands.spectrads3.*; -import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; -import com.spectralogic.ds3client.helpers.JobRecoveryException; -import com.spectralogic.ds3client.helpers.JobRecoveryNotActiveException; -import com.spectralogic.ds3client.helpers.ObjectCompletedListener; +import com.spectralogic.ds3client.helpers.*; import com.spectralogic.ds3client.helpers.options.WriteJobOptions; import com.spectralogic.ds3client.integration.test.helpers.JobStatusHelper; import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds; @@ -1178,7 +1175,7 @@ public void testHelperMetadata() throws IOException, URISyntaxException { final AtomicBoolean calledWithMetadata = new AtomicBoolean(false); - job.withMetadata(new Ds3ClientHelpers.MetadataAccess() { + job.withMetadata(new MetadataAccess() { @Override public Map getMetadataValue(final String filename) { if (filename.equals("beowulf.txt")) { diff --git a/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java b/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java index 0ab05d727..6d1c7fe6a 100644 --- a/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java +++ b/ds3-sdk-samples/src/main/java/com/spectralogic/ds3client/samples/BulkPutWithMetadata.java @@ -19,6 +19,7 @@ import com.spectralogic.ds3client.Ds3ClientBuilder; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers; import com.spectralogic.ds3client.helpers.FileObjectPutter; +import com.spectralogic.ds3client.helpers.MetadataAccess; import com.spectralogic.ds3client.models.bulk.Ds3Object; import java.io.IOException; @@ -51,7 +52,7 @@ public static void main(final String[] args) throws IOException { final Ds3ClientHelpers.Job job = helper.startWriteJob(bucketName, objects); // To put metadata with each file we need to attach the metadata with a callback - job.withMetadata(new Ds3ClientHelpers.MetadataAccess() { + job.withMetadata(new MetadataAccess() { @Override public Map getMetadataValue(final String objectName) { // Return a map of the metadata that you want assigned to the request object diff --git a/ds3-sdk/build.gradle b/ds3-sdk/build.gradle index e362f20c0..a43a83622 100644 --- a/ds3-sdk/build.gradle +++ b/ds3-sdk/build.gradle @@ -77,7 +77,6 @@ jar.dependsOn genConfigProperties dependencies { compile 'org.apache.httpcomponents:httpclient:4.5.1' - compile 'commons-io:commons-io:2.4' compile 'org.codehaus.woodstox:woodstox-core-asl:4.4.1' compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.7.1' compile 'com.google.guava:guava:20.0' diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java index 45c734ef6..c029e1ce1 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java @@ -29,7 +29,6 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.Path; import java.util.Collection; -import java.util.Map; import java.util.UUID; /** @@ -121,10 +120,6 @@ void transfer(final ObjectChannelBuilder channelBuilder) throws IOException; } - public interface MetadataAccess { - Map getMetadataValue(final String filename); - } - /** * Wraps the given {@link com.spectralogic.ds3client.Ds3ClientImpl} with helper methods. * @param client An instance of {@link com.spectralogic.ds3client.Ds3Client}, usually gotten from a call to diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java index e2ae8fd80..940d14290 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java @@ -83,7 +83,7 @@ public void removeMetadataReceivedListener(final MetadataReceivedListener listen } @Override - public Ds3ClientHelpers.Job withMetadata(final Ds3ClientHelpers.MetadataAccess access) { + public Ds3ClientHelpers.Job withMetadata(final MetadataAccess access) { throw new IllegalStateException("withMetadata method is not used with Read Jobs"); } diff --git a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java index 34464d855..287be1d8d 100644 --- a/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java +++ b/ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java @@ -50,7 +50,7 @@ class WriteJobImpl extends JobImpl { private final int retryAfter; // Negative retryAfter value represent infinity retries private final int retryDelay; //Negative value means use default - private Ds3ClientHelpers.MetadataAccess metadataAccess = null; + private MetadataAccess metadataAccess = null; private ChecksumFunction checksumFunction = null; public WriteJobImpl( @@ -80,7 +80,7 @@ public void removeMetadataReceivedListener(final MetadataReceivedListener listen } @Override - public Ds3ClientHelpers.Job withMetadata(final Ds3ClientHelpers.MetadataAccess access) { + public Ds3ClientHelpers.Job withMetadata(final MetadataAccess access) { checkRunning(); this.metadataAccess = access; return this;