add initial Swift support #3747

This branch was created based on on 4.5-export-harvest-swift which
recently had code from ferrys:4.5-export-harvest-swift-update merged
into it. This new branch was not a straight merge, however, because
there was some cruft in it that we don't want in the git history. The
latest from develop was merged into this new
3747-swift-and-compute-button branch and a `git reset` was performed to
make a single commit with all of the changes (and only the changes) that
we think we want. This single commit is being made by @pdurbin but thank
you to @ferrys @anuj-rt and @landreev who did all the actual work!

pom.xml

@@ -319,17 +319,31 @@
             <artifactId>jacoco-maven-plugin</artifactId>
             <version>0.7.5.201505241946</version>
         </dependency>
+        <!-- Added specific versionf of SLF4J jars; otherwise we end up with 
+        conflicting versions of slf4j-api and slf4j-log4j12, loaded as
+        dependencies for different packages; which was causing some incompatibility
+        issues -L.A. 4.3 -->  
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+            <version>1.7.7</version>
+	</dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
             <version>1.10.19</version>
         </dependency>
         <!-- Added for DataCite -->
-        <dependency>
+        <!--dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
             <version>1.6.1</version>
-        </dependency>
+        </dependency -->
         <dependency>
             <groupId>axis</groupId>
             <artifactId>axis</artifactId>
@@ -355,6 +369,12 @@
             <artifactId>commons-codec</artifactId>
             <version>1.9</version>
         </dependency>
+	<!-- JavaSwift/JOSS: for accessing OpenStack cloud storage -->
+	<dependency>
+            <groupId>org.javaswift</groupId>
+            <artifactId>joss</artifactId>
+            <version>0.9.10</version>
+        </dependency>
         <dependency>
             <groupId>com.maxmind.geoip2</groupId>
             <artifactId>geoip2</artifactId>

src/main/java/edu/harvard/iq/dataverse/DataFile.java

@@ -567,7 +567,7 @@ public String getOriginalChecksumType() {
     }
 
     public DataFileIO getAccessObject() throws IOException {
-        DataFileIO dataAccess =  DataAccess.createDataAccessObject(this);
+        DataFileIO dataAccess =  DataAccess.getDataFileIO(this);
 
         if (dataAccess == null) {
             throw new IOException("Failed to create access object for datafile.");

src/main/java/edu/harvard/iq/dataverse/FilePage.java

@@ -6,6 +6,7 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.DatasetVersionServiceBean.RetrieveDatasetVersionResponse;
+import edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.datasetutility.TwoRavensHelper;
@@ -616,7 +617,20 @@ public boolean isPubliclyDownloadable() {
     }
 
     public String getPublicDownloadUrl() {
-        return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId);
+        if (System.getProperty("dataverse.files.storage-driver-id").equals("swift")) {
+            String fileDownloadUrl = null;
+            try {
+                SwiftAccessIO swiftIO = (SwiftAccessIO) getFile().getAccessObject();
+                swiftIO.open();
+                fileDownloadUrl = swiftIO.getRemoteUrl();
+                logger.info("Swift url: " + fileDownloadUrl);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            return fileDownloadUrl;
+        } else {
+            return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId);
+        }
     }
 
 }

src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java

@@ -56,7 +56,7 @@ public void writeTo(BundleDownloadInstance di, Class<?> clazz, Type type, Annota
             if (di.getDownloadInfo() != null && di.getDownloadInfo().getDataFile() != null) {
                 DataAccessRequest daReq = new DataAccessRequest();
                 DataFile sf = di.getDownloadInfo().getDataFile();
-                DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq);
+                DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq);
 
                 if (accessObject != null) {
                     accessObject.open();

src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java

@@ -64,7 +64,7 @@ public void writeTo(DownloadInstance di, Class<?> clazz, Type type, Annotation[]
 
 
             DataFile sf = di.getDownloadInfo().getDataFile();
-            DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq);
+            DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq);
 
             if (accessObject != null) {
                 accessObject.open();

src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java

@@ -21,8 +21,8 @@
 package edu.harvard.iq.dataverse.dataaccess;
 
 import edu.harvard.iq.dataverse.DataFile;
-
 import java.io.IOException;
+import org.javaswift.joss.model.StoredObject;
 
 /**
  *
@@ -34,21 +34,35 @@ public DataAccess() {
 
     }
 
-    public static DataFileIO createDataAccessObject (DataFile df) throws IOException {
-        return createDataAccessObject (df, null);
+    // set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift
+    public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id");
+    public static final String DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS = System.getProperty("dataverse.files.swift-endpoint-start");
+    public static String swiftFileUri;
+    public static String swiftContainerUri;
+
+    // The getDataFileIO() methods initialize DataFileIO objects for
+    // datafiles that are already saved using one of the supported Dataverse
+    // DataAccess IO drivers.
+    public static DataFileIO getDataFileIO(DataFile df) throws IOException {
+        return getDataFileIO(df, null);
     }
 
-    public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest req) throws IOException {
+    public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throws IOException {
 
-        if (df == null ||
-                df.getStorageIdentifier() == null ||
-                df.getStorageIdentifier().equals("")) {
-            throw new IOException ("createDataAccessObject: null or invalid datafile.");
+        if (df == null
+                || df.getStorageIdentifier() == null
+                || df.getStorageIdentifier().equals("")) {
+            throw new IOException("getDataAccessObject: null or invalid datafile.");
         }
 
         if (df.getStorageIdentifier().startsWith("file://")
                 || (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) {
             return new FileAccessIO (df, req);
+        } else if (df.getStorageIdentifier().startsWith("swift://")
+                || df.getStorageIdentifier().startsWith(DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) {
+            return new SwiftAccessIO(df, req);
+        } else if (df.getStorageIdentifier().startsWith("tmp://")) {
+            throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet.");
         }
 
         // No other storage methods are supported as of now! -- 4.0.1
@@ -58,6 +72,55 @@ public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest
         // "storage identifier". 
         // -- L.A. 4.0.2
 
-        throw new IOException ("createDataAccessObject: Unsupported storage method.");
+        throw new IOException("getDataAccessObject: Unsupported storage method.");
+    }
+
+    // createDataAccessObject() methods create a *new*, empty DataAccess objects,
+    // for saving new, not yet saved datafiles.
+    public static DataFileIO createNewDataFileIO(DataFile df, String storageTag) throws IOException {
+
+        return createNewDataFileIO(df, storageTag, DEFAULT_STORAGE_DRIVER_IDENTIFIER);
     }
+
+    public static DataFileIO createNewDataFileIO(DataFile df, String storageTag, String driverIdentifier) throws IOException {
+        if (df == null
+                || storageTag == null
+                || storageTag.equals("")) {
+            throw new IOException("getDataAccessObject: null or invalid datafile.");
+        }
+
+        DataFileIO dataFileIO = null;
+
+        df.setStorageIdentifier(storageTag);
+
+        if (driverIdentifier.equals("file")) {
+            dataFileIO = new FileAccessIO(df, null);
+        } else if (driverIdentifier.equals("swift")) {
+            dataFileIO = new SwiftAccessIO(df, null);
+        } else {
+            throw new IOException("createDataAccessObject: Unsupported storage method " + driverIdentifier);
+        }
+
+        dataFileIO.open(DataAccessOption.WRITE_ACCESS);
+        return dataFileIO;
+    }
+
+    public static String getSwiftFileURI(StoredObject fileObject) throws IOException {
+        String fileUri;
+        try {
+            fileUri = fileObject.getPublicURL();
+        } catch (Exception ex) {
+            ex.printStackTrace();
+            throw new IOException("SwiftAccessIO: failed to get file storage location");
+        }
+        return fileUri;
+    }
+
+    public static String getSwiftContainerURI(StoredObject fileObject) throws IOException {
+        String containerUri;
+        containerUri = getSwiftFileURI(fileObject);
+        containerUri = containerUri.substring(0, containerUri.lastIndexOf('/'));
+        return containerUri;
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java

@@ -221,7 +221,7 @@ public long addFileToZipStream(DataFile dataFile) throws IOException {
         boolean createManifest = fileManifest != null;
 
         DataAccessRequest daReq = new DataAccessRequest();
-        DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq);
+        DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq);
 
         if (accessObject != null) {
             accessObject.open();