Metadata updates

ds3-interfaces/build.gradle

@@ -22,7 +22,12 @@ buildscript {
 
 apply plugin: 'com.github.johnrengelman.shadow'
 
+dependencies {
+    compile 'commons-io:commons-io:2.4'
+}
+
 shadowJar {
+    relocate 'org.apache', 'ds3fatjar.org.apache'
     dependencies {
         exclude(dependency('org.hamcrest:hamcrest-library:1.3'))
         exclude(dependency('org.mockito:mockito-core:1.10.19'))

ds3-interfaces/src/main/java/com/spectralogic/ds3client/exceptions/AggregateException.java

@@ -0,0 +1,60 @@
+/*
+ * ****************************************************************************
+ *    Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *    Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *    this file except in compliance with the License. A copy of the License is located at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file.
+ *    This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *    CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *    specific language governing permissions and limitations under the License.
+ *  ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.exceptions;
+
+import org.apache.commons.io.output.StringBuilderWriter;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+public class AggregateException extends RuntimeException {
+
+    private final List<Throwable> exceptions;
+
+    public AggregateException(final Iterable<Throwable> exceptions) {
+        super("One or more exceptions were aggregated together");
+        this.exceptions = exceptionList(exceptions);
+    }
+
+    private static List<Throwable> exceptionList(final Iterable<Throwable> exceptions) {
+        final List<Throwable> exceptionList = new ArrayList<>();
+        for (final Throwable t : exceptions) {
+            exceptionList.add(t);
+        }
+        return exceptionList;
+    }
+
+    public Iterable<Throwable> getExceptions() {
+        return exceptions;
+    }
+
+    public String toString() {
+        final StringBuilder builder = new StringBuilder();
+        builder.append("One or more exceptions were aggregated:");
+
+        try (final StringBuilderWriter writer = new StringBuilderWriter(builder);
+             final PrintWriter printWriter = new PrintWriter(writer)) {
+
+            for (final Throwable t : exceptions) {
+                printWriter.append("Exception: ").append(t.getMessage());
+                t.printStackTrace(printWriter);
+            }
+        }
+
+        return builder.toString();
+    }
+}

ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers/MetadataAccess.java → ds3-interfaces/src/main/java/com/spectralogic/ds3client/helpers/MetadataAccess.java

@@ -13,7 +13,7 @@
  * ****************************************************************************
  */
 
-package com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+package com.spectralogic.ds3client.helpers;
 
 import java.util.Map;
 

ds3-interfaces/src/test/java/com/spectralogic/ds3client/exceptions/AggregateException_Test.java

@@ -0,0 +1,73 @@
+/*
+ * ****************************************************************************
+ *    Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
+ *    Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *    this file except in compliance with the License. A copy of the License is located at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file.
+ *    This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *    CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *    specific language governing permissions and limitations under the License.
+ *  ****************************************************************************
+ */
+
+package com.spectralogic.ds3client.exceptions;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class AggregateException_Test {
+
+    @Test
+    public void basicAggregate() {
+        final Exception e1 = new Exception("first exception");
+        final Exception e2 = new Exception("second exception");
+
+        final List<Throwable> exceptionList = new ArrayList<>();
+        exceptionList.add(e1);
+        exceptionList.add(e2);
+
+        final AggregateException aggregateException = new AggregateException(exceptionList);
+
+        final Iterator<Throwable> exceptionIter = aggregateException.getExceptions().iterator();
+
+        assertTrue(exceptionIter.hasNext());
+        assertThat(exceptionIter.next(), is(notNullValue()));
+        assertTrue(exceptionIter.hasNext());
+        assertThat(exceptionIter.next(), is(notNullValue()));
+        assertFalse(exceptionIter.hasNext());
+    }
+
+    @Test
+    public void repeatIteration() {
+        final Exception e1 = new Exception("first exception");
+
+        final List<Throwable> exceptionList = new ArrayList<>();
+        exceptionList.add(e1);
+
+        final AggregateException aggregateException = new AggregateException(exceptionList);
+
+        final Iterator<Throwable> exceptionIter = aggregateException.getExceptions().iterator();
+
+        assertTrue(exceptionIter.hasNext());
+        assertThat(exceptionIter.next(), is(notNullValue()));
+        assertFalse(exceptionIter.hasNext());
+
+        final Iterator<Throwable> repeatIter = aggregateException.getExceptions().iterator();
+        assertTrue(repeatIter.hasNext());
+        assertThat(repeatIter.next(), is(notNullValue()));
+        assertFalse(repeatIter.hasNext());
+
+    }
+}

ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataAccessImpl.java

@@ -17,7 +17,7 @@
 
 
 import com.google.common.collect.ImmutableMap;
-import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.MetadataAccess;
+import com.spectralogic.ds3client.helpers.MetadataAccess;
 import com.spectralogic.ds3client.metadata.interfaces.MetadataStore;
 import com.spectralogic.ds3client.utils.Platform;
 

ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/MetadataReceivedListenerImpl.java

@@ -15,15 +15,24 @@
 
 package com.spectralogic.ds3client.metadata;
 
+import com.google.common.collect.ImmutableList;
+import com.spectralogic.ds3client.exceptions.AggregateException;
 import com.spectralogic.ds3client.helpers.MetadataReceivedListener;
 import com.spectralogic.ds3client.metadata.interfaces.MetadataRestore;
 import com.spectralogic.ds3client.networking.Metadata;
+import com.spectralogic.ds3client.utils.Guard;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
 public class MetadataReceivedListenerImpl implements MetadataReceivedListener {
+
+    private final static Logger LOG = LoggerFactory.getLogger(MetadataReceivedListenerImpl.class);
+
     private final String localFilePath;
 
+
     public MetadataReceivedListenerImpl(final String localFilePath) {
         this.localFilePath = localFilePath;
     }
@@ -45,17 +54,41 @@ public void metadataReceived(final String filename, final Metadata metadata) {
      * @param metadata   metadata which needs to be set on local file
      */
     private void restoreMetaData(final String objectName, final Metadata metadata) throws IOException, InterruptedException {
+
+        final ImmutableList.Builder<Throwable> exceptionBuilder = ImmutableList.builder();
+
         //get metadatarestore on the basis of os
         final MetadataRestore metadataRestore = new MetadataRestoreFactory().getOSSpecificMetadataRestore(metadata, objectName);
         //restore os name
         metadataRestore.restoreOSName();
+
         //restore user and owner based on OS
-        metadataRestore.restoreUserAndOwner();
+        try {
+            metadataRestore.restoreUserAndOwner();
+        } catch (final Throwable t) {
+             LOG.error("Could not restore owner and owner information", t);
+             exceptionBuilder.add(t);
+        }
+
         //restore creation and modified time based on OS
-        metadataRestore.restoreFileTimes();
-        //restore permissions based on OS
-        metadataRestore.restorePermissions();
-    }
+        try {
+            metadataRestore.restoreFileTimes();
+        } catch (final Throwable t) {
+            LOG.error("Could not restore the file times", t);
+            exceptionBuilder.add(t);
+        }
 
+        //restore permissions based on OS
+        try {
+            metadataRestore.restorePermissions();
+        } catch (final Throwable t) {
+            LOG.error("Could not restore the file permissions", t);
+            exceptionBuilder.add(t);
+        }
 
+        final ImmutableList<Throwable> exceptions = exceptionBuilder.build();
+        if (!Guard.isNullOrEmpty(exceptions)) {
+            throw new AggregateException(exceptions);
+        }
+    }
 }

ds3-metadata/src/main/java/com/spectralogic/ds3client/metadata/WindowsMetadataRestore.java

@@ -18,7 +18,11 @@
 import com.google.common.collect.ImmutableList;
 import com.spectralogic.ds3client.metadata.jna.Advapi32;
 import com.spectralogic.ds3client.networking.Metadata;
+import com.spectralogic.ds3client.utils.Guard;
 import com.sun.jna.platform.win32.WinNT;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileSystems;
@@ -36,6 +40,9 @@
 import static com.spectralogic.ds3client.metadata.MetadataKeyConstants.KEY_OWNER;
 
 class WindowsMetadataRestore extends AbstractMetadataRestore {
+
+    private static final Logger LOG = LoggerFactory.getLogger(WindowsMetadataRestore.class);
+
     WindowsMetadataRestore(final Metadata metadata, final String filePath, final String localOS) {
         this.metadata = metadata;
         this.objectName = filePath;
@@ -44,26 +51,31 @@ class WindowsMetadataRestore extends AbstractMetadataRestore {
 
     @Override
     public void restoreUserAndOwner() throws IOException {
-        if (storedOS.equals(localOS)) {
-            String ownerSid = null;
-            if (metadata.get(KEY_OWNER).size() > 0) {
-                ownerSid = metadata.get(KEY_OWNER).get(0);
-            }
-            String groupSid = null;
-            if (metadata.get(KEY_GROUP).size() > 0) {
-                groupSid = metadata.get(KEY_GROUP).get(0);
-            }
-            if (ownerSid != null && groupSid != null && !ownerSid.equals("") && !groupSid.equals("")) {
-                setOwnerIdandGroupId(ownerSid, groupSid);
+        if (storedOS != null && storedOS.equals(localOS)) {
+
+            final String ownerSid = getMetadataProperty(metadata, KEY_OWNER);
+            final String groupSid = getMetadataProperty(metadata, KEY_GROUP);
+
+            if (!Guard.isStringNullOrEmpty(ownerSid) && !Guard.isStringNullOrEmpty(groupSid)) {
+                setOwnerIdAndGroupId(ownerSid, groupSid);
+            } else {
+                LOG.warn("Cannot determine owner or group settings for {}", this.objectName);
             }
+        } else {
+            LOG.warn("The OS settings for owner and group properties cannot be restored for {}", this.objectName);
         }
+    }
 
+    private static String getMetadataProperty(final Metadata metadata, final String metadataName) {
+        return metadata.get(metadataName).get(0);
     }
 
     @Override
     public void restorePermissions() throws IOException, InterruptedException {
         if (storedOS != null && storedOS.equals(localOS)) {
             setPermissionsForWindows();
+        } else {
+            LOG.warn("The OS settings for the file permissions cannot be restored for {}", this.objectName);
         }
         restoreFlags();
     }
@@ -76,19 +88,26 @@ private void setPermissionsForWindows() throws IOException {
         final String userListDisplay;
         final String[] users;
         final String[] usersDisplay;
-        if (metadata.get("ds3-userList").size() > 0) {
+        if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-userList"))) {
             userList = metadata.get("ds3-userList").get(0);
-            if (metadata.get("ds3-userListDisplay").size() > 0) {
+            if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-userListDisplay"))) {
                 userListDisplay = metadata.get("ds3-userListDisplay").get(0);
                 users = userList.split("-");
                 usersDisplay = userListDisplay.split("-");
                 for (int i = 0; i < users.length; i++) {
-                    if (metadata.get("ds3-" + users[i]).size() > 0) {
-                        final String ownerPermission = metadata.get("ds3-" + users[i]).get(0);
+                    final String user = users[i];
+                    if (Guard.isNotNullAndNotEmpty(metadata.get("ds3-" + user))) {
+                        final String ownerPermission = metadata.get("ds3-" + user).get(0);
                         restorePermissionByUser(ownerPermission, usersDisplay[i], aclEntryBuilder);
+                    } else {
+                        LOG.warn("Did not find any permissions for {} for file {}", user, this.objectName);
                     }
                 }
+            } else {
+                LOG.warn("There was not a 'userListDisplay' metadata entry for file {}, so we will not restore any permissions", this.objectName);
             }
+        } else {
+            LOG.warn("There was not a 'userList' metadata entry for file {}, so we will not restore any permissions", this.objectName);
         }
 
         aclAttributeView.setAcl(aclEntryBuilder.build());
@@ -148,7 +167,7 @@ private void restorePermissionByUser(final String permission,
      * @param ownerSidId sid of the owner
      * @param groupSidId sid of the group
      */
-    private void setOwnerIdandGroupId(final String ownerSidId, final String groupSidId) throws IOException {
+    private void setOwnerIdAndGroupId(final String ownerSidId, final String groupSidId) throws IOException {
         final int infoType = WinNT.OWNER_SECURITY_INFORMATION | WinNT.GROUP_SECURITY_INFORMATION;
         final WinNT.PSIDByReference referenceOwner = new WinNT.PSIDByReference();
         Advapi32.INSTANCE.ConvertStringSidToSid(ownerSidId, referenceOwner);
@@ -166,7 +185,11 @@ private void restoreFlags()  throws IOException, InterruptedException {
             if (metadata.get(KEY_FLAGS).size() > 0) {
                 final String flags = metadata.get(KEY_FLAGS).get(0);
                 restoreFlagsWindows(flags);
+            } else {
+                LOG.warn("The file flag settings do not exist for file {} and will not be restored", this.objectName);
             }
+        } else {
+            LOG.warn("The OS settings for restoring the file flags cannot be done for {}", this.objectName);
         }
     }
 
@@ -199,9 +222,12 @@ private void restoreFlagsWindows(final String flag) throws IOException, Interrup
             stringBuilder.append(" -I");
             stringBuilder.append(" -H");
         }
-        stringBuilder.append(" " + "\"" + objectName + "\"");
+        stringBuilder.append(" " + "\"").append(objectName).append("\"");
 
         final Process p = Runtime.getRuntime().exec(stringBuilder.toString().split(" "));
-        p.waitFor();
+        final int returnCode = p.waitFor();
+        if (returnCode != 0) {
+            LOG.error("Restoring the flag settings for file {} was not successful", this.objectName);
+        }
     }
 }

ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Smoke_Test.java

@@ -22,10 +22,7 @@
 import com.spectralogic.ds3client.commands.*;
 import com.spectralogic.ds3client.commands.interfaces.BulkResponse;
 import com.spectralogic.ds3client.commands.spectrads3.*;
-import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
-import com.spectralogic.ds3client.helpers.JobRecoveryException;
-import com.spectralogic.ds3client.helpers.JobRecoveryNotActiveException;
-import com.spectralogic.ds3client.helpers.ObjectCompletedListener;
+import com.spectralogic.ds3client.helpers.*;
 import com.spectralogic.ds3client.helpers.options.WriteJobOptions;
 import com.spectralogic.ds3client.integration.test.helpers.JobStatusHelper;
 import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
@@ -1178,7 +1175,7 @@ public void testHelperMetadata() throws IOException, URISyntaxException {
 
             final AtomicBoolean calledWithMetadata = new AtomicBoolean(false);
 
-            job.withMetadata(new Ds3ClientHelpers.MetadataAccess() {
+            job.withMetadata(new MetadataAccess() {
                 @Override
                 public Map<String, String> getMetadataValue(final String filename) {
                     if (filename.equals("beowulf.txt")) {