diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 430a5fb9ec9..9fd8511c071 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -149,6 +149,29 @@ Enabling a second authentication provider will result in the Log In page showing - ``:AllowSignUp`` is set to "false" per the :doc:`config` section to prevent users from creating local accounts via the web interface. Please note that local accounts can also be created via API, and the way to prevent this is to block the ``builtin-users`` endpoint or scramble (or remove) the ``BuiltinUsers.KEY`` database setting per the :doc:`config` section. - The "builtin" authentication provider has been disabled. Note that disabling the builting auth provider means that the API endpoint for converting an account from a remote auth provider will not work. This is the main reason why https://github.com/IQSS/dataverse/issues/2974 is still open. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to builtin. Then the user initiates a conversion from builtin to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. +Swift Installation +------------------ + +On every generic Dataverse installation, datafiles are stored in local storage. You can opt for a experimental dataverse installation with a Swift Object Storage backend instead. Each dataset that you create is saved as a container. Each datafile is saved as a file within the container. + +In order to configure a Swift installation, there are two steps you need to complete before running the installer: + +First, you need to add a file called swift.properties in ``/glassfish4/glassfish/domains/domain1/config``. It needs to be configured as follows: + +.. code-block:: none + swift.default.endpoint=endpoint1 + swift.auth_url.endpoint1=your-auth-url + swift.tenant.endpoint1=your-tenant-name + swift.username.endpoint1=your-username + swift.password.endpoint1=your-password + swift.auth_type.endpoint1=your-authentication-type + swift.swift_endpoint.endpoint1=your-swift-endpoint + +where authentication type can either be "keystone" (without the quotes) or it will assumed to be basic. Additionally, authentication url should be your keystone authentication url which includes the tokens. For example, https://dataverse.org:35357/v2.0/tokens. + +Second, you need to update the JVM option ``dataverse.files.storage-driver-id``. You can do this by running the delete command ``./asadmin $ASADMIN_OPTS delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"`` and then the create command ``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.storage-driver-id=swift"``. + + JVM Options ----------- @@ -188,6 +211,11 @@ dataverse.files.directory This is how you configure the path to which files uploaded by users are stored. +dataverse.files.storage-driver-id ++++++++++++++++++++++++++++++++++ + +Initialized to file, this is what you would change if you were interested in a Swift backend for your installation. See above in the Swift Installation section. + dataverse.auth.password-reset-timeout-in-minutes ++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/pom.xml b/pom.xml index d1bb280f86e..37af517f883 100644 --- a/pom.xml +++ b/pom.xml @@ -319,17 +319,31 @@ jacoco-maven-plugin 0.7.5.201505241946 + + + org.slf4j + slf4j-api + 1.7.7 + + + org.slf4j + slf4j-log4j12 + 1.7.7 + org.mockito mockito-core 1.10.19 - + axis axis @@ -355,6 +369,12 @@ commons-codec 1.9 + + + org.javaswift + joss + 0.9.10 + com.maxmind.geoip2 geoip2 diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index dfba4ca684d..6b415542c99 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -568,7 +568,7 @@ public String getOriginalChecksumType() { } public DataFileIO getAccessObject() throws IOException { - DataFileIO dataAccess = DataAccess.createDataAccessObject(this); + DataFileIO dataAccess = DataAccess.getDataFileIO(this); if (dataAccess == null) { throw new IOException("Failed to create access object for datafile."); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 08479dfcd6f..fe5e7bfd213 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.DatasetVersionServiceBean.RetrieveDatasetVersionResponse; +import edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.datasetutility.TwoRavensHelper; @@ -616,7 +617,20 @@ public boolean isPubliclyDownloadable() { } public String getPublicDownloadUrl() { - return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); + if (System.getProperty("dataverse.files.storage-driver-id").equals("swift")) { + String fileDownloadUrl = null; + try { + SwiftAccessIO swiftIO = (SwiftAccessIO) getFile().getAccessObject(); + swiftIO.open(); + fileDownloadUrl = swiftIO.getRemoteUrl(); + logger.info("Swift url: " + fileDownloadUrl); + } catch (Exception e) { + e.printStackTrace(); + } + return fileDownloadUrl; + } else { + return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); + } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java index f4070e9b4e5..7e32d4ae231 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java @@ -56,7 +56,7 @@ public void writeTo(BundleDownloadInstance di, Class clazz, Type type, Annota if (di.getDownloadInfo() != null && di.getDownloadInfo().getDataFile() != null) { DataAccessRequest daReq = new DataAccessRequest(); DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index c611cd6e796..2d5008d54f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -64,7 +64,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index c4a64bff350..f8483dfa9ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -21,8 +21,8 @@ package edu.harvard.iq.dataverse.dataaccess; import edu.harvard.iq.dataverse.DataFile; - import java.io.IOException; +import org.javaswift.joss.model.StoredObject; /** * @@ -34,21 +34,32 @@ public DataAccess() { } - public static DataFileIO createDataAccessObject (DataFile df) throws IOException { - return createDataAccessObject (df, null); + // set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift + public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); + public static String swiftFileUri; + + // The getDataFileIO() methods initialize DataFileIO objects for + // datafiles that are already saved using one of the supported Dataverse + // DataAccess IO drivers. + public static DataFileIO getDataFileIO(DataFile df) throws IOException { + return getDataFileIO(df, null); } - public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest req) throws IOException { + public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throws IOException { - if (df == null || - df.getStorageIdentifier() == null || - df.getStorageIdentifier().equals("")) { - throw new IOException ("createDataAccessObject: null or invalid datafile."); + if (df == null + || df.getStorageIdentifier() == null + || df.getStorageIdentifier().equals("")) { + throw new IOException("getDataAccessObject: null or invalid datafile."); } if (df.getStorageIdentifier().startsWith("file://") || (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) { return new FileAccessIO (df, req); + } else if (df.getStorageIdentifier().startsWith("swift://")){ + return new SwiftAccessIO(df, req); + } else if (df.getStorageIdentifier().startsWith("tmp://")) { + throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet."); } // No other storage methods are supported as of now! -- 4.0.1 @@ -58,6 +69,54 @@ public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest // "storage identifier". // -- L.A. 4.0.2 - throw new IOException ("createDataAccessObject: Unsupported storage method."); + throw new IOException("getDataAccessObject: Unsupported storage method."); + } + + // createDataAccessObject() methods create a *new*, empty DataAccess objects, + // for saving new, not yet saved datafiles. + public static DataFileIO createNewDataFileIO(DataFile df, String storageTag) throws IOException { + + return createNewDataFileIO(df, storageTag, DEFAULT_STORAGE_DRIVER_IDENTIFIER); } + + public static DataFileIO createNewDataFileIO(DataFile df, String storageTag, String driverIdentifier) throws IOException { + if (df == null + || storageTag == null + || storageTag.equals("")) { + throw new IOException("getDataAccessObject: null or invalid datafile."); + } + + DataFileIO dataFileIO = null; + + df.setStorageIdentifier(storageTag); + + if (driverIdentifier == null) { + driverIdentifier = "file"; + } + + if (driverIdentifier.equals("file")) { + dataFileIO = new FileAccessIO(df, null); + } else if (driverIdentifier.equals("swift")) { + dataFileIO = new SwiftAccessIO(df, null); + } else { + throw new IOException("createDataAccessObject: Unsupported storage method " + driverIdentifier); + } + + dataFileIO.open(DataAccessOption.WRITE_ACCESS); + return dataFileIO; + } + + public static String getSwiftFileURI(StoredObject fileObject) throws IOException { + String fileUri; + try { + fileUri = fileObject.getPublicURL(); + } catch (Exception ex) { + ex.printStackTrace(); + throw new IOException("SwiftAccessIO: failed to get file storage location"); + } + return fileUri; + } + + + } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java index e75f326c3fa..d9022d87654 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java @@ -221,7 +221,7 @@ public long addFileToZipStream(DataFile dataFile) throws IOException { boolean createManifest = fileManifest != null; DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index 419b6babfc4..c4ce2dfc532 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -152,6 +152,9 @@ public static String getImageThumbAsBase64(DataFile file, int size) { try { fileAccess = (FileAccessIO) file.getAccessObject(); + } catch (ClassCastException ex) { + logger.warning("Unable to cast file id " + file.getId() + " from DataFileIO to FileAccessIO."); + return null; } catch (IOException ex) { // too bad - but not fatal logger.warning("getImageThumbAsBase64: Failed to obtain FileAccess object for DataFile id " + file.getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java new file mode 100644 index 00000000000..951e0c21663 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -0,0 +1,432 @@ +package edu.harvard.iq.dataverse.dataaccess; + +import edu.harvard.iq.dataverse.DataFile; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channel; +import java.nio.channels.Channels; +import java.nio.file.Path; +import java.util.Properties; +import java.util.logging.Logger; +import org.javaswift.joss.client.factory.AccountFactory; +import static org.javaswift.joss.client.factory.AuthenticationMethod.BASIC; +import org.javaswift.joss.model.Account; +import org.javaswift.joss.model.Container; +import org.javaswift.joss.model.StoredObject; + +/** + * + * @author leonid andreev + */ +/* + Experimental Swift driver, implemented as part of the Dataverse - Mass Open Cloud + collaboration. + Read-only access, for now. + */ +public class SwiftAccessIO extends DataFileIO { + + private String swiftFolderPath; + private String swiftFileName = null; + + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO"); + + public SwiftAccessIO() throws IOException { + this(null); + } + + public SwiftAccessIO(DataFile dataFile) throws IOException { + this(dataFile, null); + + } + + public SwiftAccessIO(DataFile dataFile, DataAccessRequest req) throws IOException { + + super(dataFile, req); + + this.setIsLocalFile(false); + } + + private boolean isReadAccess = false; + private boolean isWriteAccess = false; + private Properties swiftProperties = null; + private Account account = null; + private StoredObject swiftFileObject = null; + + @Override + public boolean canRead() { + return isReadAccess; + } + + @Override + public boolean canWrite() { + return isWriteAccess; + } + + @Override + public void open(DataAccessOption... options) throws IOException { + + DataFile dataFile = this.getDataFile(); + DataAccessRequest req = this.getRequest(); + + if (req != null && req.getParameter("noVarHeader") != null) { + this.setNoVarHeader(true); + } + + if (isWriteAccessRequested(options)) { + isWriteAccess = true; + isReadAccess = false; + } else { + isWriteAccess = false; + isReadAccess = true; + } + + if (this.getDataFile().getStorageIdentifier() == null || "".equals(this.getDataFile().getStorageIdentifier())) { + throw new IOException("Data Access: No local storage identifier defined for this datafile."); + } + + if (isReadAccess) { + InputStream fin = openSwiftFileAsInputStream(); + + if (fin == null) { + throw new IOException("Failed to open Swift file " + getStorageLocation()); + } + + this.setInputStream(fin); + setChannel(Channels.newChannel(fin)); + + } else if (isWriteAccess) { + swiftFileObject = initializeSwiftFileObject(true); + } + + this.setMimeType(dataFile.getContentType()); + try { + this.setFileName(dataFile.getFileMetadata().getLabel()); + } catch (Exception ex) { + this.setFileName("unknown"); + } + + // This "status" is a leftover from 3.6; we don't have a use for it + // in 4.0 yet; and we may not need it at all. + // -- L.A. 4.0.2 + this.setStatus(200); + } + + // this is a Swift-specific override of the convenience method provided in the + // DataFileIO for copying a local Path (for ex., a temp file, into this DataAccess location): + + @Override + public void copyPath(Path fileSystemPath) throws IOException { + long newFileSize = -1; + + if (swiftFileObject == null || !this.canWrite()) { + open(DataAccessOption.WRITE_ACCESS); + } + + File inputFile = null; + + try { + inputFile = fileSystemPath.toFile(); + + swiftFileObject.uploadObject(inputFile); + + newFileSize = inputFile.length(); + + } catch (Exception ioex) { + String failureMsg = ioex.getMessage(); + if (failureMsg == null) { + failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject"; + } + + throw new IOException(failureMsg); + } + + // if it has uploaded successfully, we can reset the size + // of the object: + setSize(newFileSize); + + } + // @Override + // public void copyPath(Path fileSystemPath) throws IOException { + // long newFileSize = -1; + // Properties p = getSwiftProperties(); + // String swiftEndPoint = p.getProperty("swift.default.endpoint"); + // String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint); + + // if (swiftFileObject == null || !this.canWrite()) { + // open(DataAccessOption.WRITE_ACCESS); + // } + + // File inputFile = null; + + // try { + // inputFile = fileSystemPath.toFile(); + + // //@author Anuj Thakur + // swiftFileObject.uploadObject(inputFile); + // //After the files object is uploaded the identifier is changed. + // logger.info(this.swiftFileName + " " + this.swiftFolderPath); + // this.getDataFile().setStorageIdentifier( + this.swiftFolderPath + "/" + this.swiftFileName); + + // newFileSize = inputFile.length(); + + // } catch (Exception ioex) { + // String failureMsg = ioex.getMessage(); + // if (failureMsg == null) { + // failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject"; + // } + + // throw new IOException(failureMsg); + // } + + // // if it has uploaded successfully, we can reset the size + // // of the object: + // setSize(newFileSize); + + // } + + @Override + public void delete() throws IOException { + //throw new IOException("SwiftAccessIO: delete() not yet implemented in this storage driver."); + if (swiftFileObject == null) { + try { + swiftFileObject = initializeSwiftFileObject(false); + } catch (IOException ioex) { + swiftFileObject = null; + } + } + + if (swiftFileObject != null) { + swiftFileObject.delete(); + } + } + + @Override + public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException { + + throw new IOException("SwiftAccessIO: openAuxChannel() not yet implemented in this storage driver."); + } + + @Override + public boolean isAuxObjectCached(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: isAuxObjectCached() not yet implemented in this storage driver."); + } + + @Override + public long getAuxObjectSize(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: getAuxObjectSize() not yet implemented in this storage driver."); + } + + @Override + public void backupAsAux(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: backupAsAux() not yet implemented in this storage driver."); + } + + @Override + public String getStorageLocation() { + // What should this be, for a Swift file? + // A Swift URL? + // Or a Swift URL with an authenticated Auth token? + + return null; + } + + @Override + public Path getFileSystemPath() throws IOException { + throw new IOException("SwiftAccessIO: this is a remote AccessIO object, it has no local filesystem path associated with it."); + } + + // Auxilary helper methods, Swift-specific: + private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOException { + String storageIdentifier = this.getDataFile().getStorageIdentifier(); + + String swiftEndPoint = null; + String swiftContainer = null; + + + if (storageIdentifier.startsWith("swift://")) { + // This is a call on an already existing swift object. + + String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); + + if (swiftStorageTokens.length != 3) { + // bad storage identifier + throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); + } + + swiftEndPoint = swiftStorageTokens[0]; + swiftContainer = swiftStorageTokens[1]; + swiftFileName = swiftStorageTokens[2]; + + if (swiftEndPoint == null || swiftContainer == null || swiftFileName == null + || "".equals(swiftEndPoint) || "".equals(swiftContainer) || "".equals(swiftFileName)) { + // all of these things need to be specified, for this to be a valid Swift location + // identifier. + throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); + } + } else if (this.isReadAccess) { + // An attempt to call Swift driver, in a Read mode on a non-swift stored datafile + // object! + throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); + } else if (this.isWriteAccess) { + Properties p = getSwiftProperties(); + swiftEndPoint = p.getProperty("swift.default.endpoint"); + + //swiftFolderPath = this.getDataFile().getOwner().getDisplayName(); + String swiftFolderPathSeparator = "_"; + String authorityNoSlashes = this.getDataFile().getOwner().getAuthority().replace(this.getDataFile().getOwner().getDoiSeparator(), swiftFolderPathSeparator); + swiftFolderPath = this.getDataFile().getOwner().getProtocol() + swiftFolderPathSeparator + + authorityNoSlashes.replace(".", swiftFolderPathSeparator) + + swiftFolderPathSeparator + this.getDataFile().getOwner().getIdentifier(); + swiftFileName = storageIdentifier; + //swiftFileName = this.getDataFile().getDisplayName(); + //Storage Identifier is now updated after the object is uploaded on Swift. + this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftFolderPath+":"+swiftFileName); + } else { + throw new IOException("SwiftAccessIO: unknown access mode."); + } + // Authenticate with Swift: + + account = authenticateWithSwift(swiftEndPoint); + + /* + The containers created is swiftEndPoint concatenated with the swiftContainer + property. Creating container with certain names throws 'Unable to create + container' error on Openstack. + Any datafile with http://rdgw storage identifier i.e present on Object + store service endpoint already only needs to look-up for container using + just swiftContainer which is the concatenated name. + In future, a container for the endpoint can be created and for every + other swiftContainer Object Store pseudo-folder can be created, which is + not provide by the joss Java swift library as of yet. + */ + Container dataContainer; + + if (storageIdentifier.startsWith("swift://")) { + dataContainer = account.getContainer(swiftContainer); + } else { + dataContainer = account.getContainer(swiftFolderPath); //changed from swiftendpoint + } + + if (!dataContainer.exists()) { + if (writeAccess) { + dataContainer.create(); + //dataContainer.makePublic(); + } else { + // This is a fatal condition - it has to exist, if we were to + // read an existing object! + throw new IOException("SwiftAccessIO: container " + swiftContainer + " does not exist."); + } + } + + StoredObject fileObject = dataContainer.getObject(swiftFileName); + //file download url for public files + DataAccess.swiftFileUri = DataAccess.getSwiftFileURI(fileObject); + setRemoteUrl(DataAccess.getSwiftFileURI(fileObject)); + + logger.info(DataAccess.swiftFileUri + " success"); + + if (!writeAccess && !fileObject.exists()) { + throw new IOException("SwiftAccessIO: File object " + swiftFileName + " does not exist (Dataverse datafile id: " + this.getDataFile().getId()); + } + + return fileObject; + } + + private InputStream openSwiftFileAsInputStream() throws IOException { + InputStream in = null; + + swiftFileObject = initializeSwiftFileObject(false); + + in = swiftFileObject.downloadObjectAsInputStream(); + this.setSize(swiftFileObject.getContentLength()); + + return in; + } + + private Properties getSwiftProperties() throws IOException { + if (swiftProperties == null) { + String domainRoot = System.getProperties().getProperty("com.sun.aas.instanceRoot"); + String swiftPropertiesFile = domainRoot + File.separator + "config" + File.separator + "swift.properties"; + swiftProperties = new Properties(); + swiftProperties.load(new FileInputStream(new File(swiftPropertiesFile))); + } + + return swiftProperties; + } + + Account authenticateWithSwift(String swiftEndPoint) throws IOException { + + Properties p = getSwiftProperties(); + + // (this will throw an IOException, if the swift properties file + // is missing or corrupted) + String swiftEndPointAuthUrl = p.getProperty("swift.auth_url." + swiftEndPoint); + String swiftEndPointUsername = p.getProperty("swift.username." + swiftEndPoint); + String swiftEndPointSecretKey = p.getProperty("swift.password." + swiftEndPoint); + String swiftEndPointTenantName = p.getProperty("swift.tenant." + swiftEndPoint); + String swiftEndPointAuthMethod = p.getProperty("swift.auth_type." + swiftEndPoint); + + if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null + || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) { + // again, all of these things need to be defined, for this Swift endpoint to be + // accessible. + throw new IOException("SwiftAccessIO: no configuration available for endpoint " + swiftEndPoint); + } + + // Authenticate: + Account account = null; + + /* + This try { } now authenticates using either the KEYSTONE mechanism which uses + the tenant name in addition to the Username Password and AuthUrl OR the BASIC method + Also, the AuthUrl is now the identity service endpoint of MOC Openstack + environment instead of the Object store service endpoint. + */ + // Keystone vs. Basic + try { + if (swiftEndPointAuthMethod.equals("keystone")) { + account = new AccountFactory() + .setTenantName(swiftEndPointTenantName) + .setUsername(swiftEndPointUsername) + .setPassword(swiftEndPointSecretKey) + .setAuthUrl(swiftEndPointAuthUrl) + .createAccount(); + } else { // assume BASIC + account = new AccountFactory() + .setUsername(swiftEndPointUsername) + .setPassword(swiftEndPointSecretKey) + .setAuthUrl(swiftEndPointAuthUrl) + .setAuthenticationMethod(BASIC) + .createAccount(); + } + + } catch (Exception ex) { + ex.printStackTrace(); + throw new IOException("SwiftAccessIO: failed to authenticate " + swiftEndPointAuthMethod + " for the end point " + swiftEndPoint); + } + + return account; + } + + private boolean isWriteAccessRequested(DataAccessOption... options) throws IOException { + + for (DataAccessOption option : options) { + // In the future we may need to be able to open read-write + // Channels; no support, or use case for that as of now. + + if (option == DataAccessOption.READ_ACCESS) { + return false; + } + + if (option == DataAccessOption.WRITE_ACCESS) { + return true; + } + } + + // By default, we open the file in read mode: + return false; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index f9ab137a603..be538442d22 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -37,6 +37,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.DataFileIO; import edu.harvard.iq.dataverse.dataaccess.FileAccessIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -267,12 +268,14 @@ public void addFiles (DatasetVersion version, List newFiles) { try { - DataFileIO dataAccess = dataFile.getAccessObject(); + logger.info("Attempting to create a new DataFileIO object for " + storageId); + DataFileIO dataAccess = DataAccess.createNewDataFileIO(dataFile, storageId); if (dataAccess.isLocalFile()) { localFile = true; } - + + logger.info("Successfully created a new DataFileIO object."); /* This commented-out code demonstrates how to copy bytes from a local InputStream (or a readChannel) into the @@ -311,7 +314,7 @@ from a local InputStream (or a readChannel) into the savedSuccess = true; } catch (IOException ioex) { - logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier()); + logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); } finally { if (readChannel != null) {try{readChannel.close();}catch(IOException e){}} if (writeChannel != null) {try{writeChannel.close();}catch(IOException e){}} diff --git a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java index 791b376fb1e..df81d0328aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java +++ b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java @@ -564,7 +564,7 @@ public File runDataPreprocessing(DataFile dataFile) { // send the tabular data file to the Rserve side: DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); if (accessObject == null) { return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 0211fedf0e7..fbe67d4c6e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -222,8 +222,11 @@ public enum Key { Default is false; */ TwoRavensTabularView, - - + /** + * Whether to save data files locally in files or store them in swift + * storage, default is file. + */ + Driver, /** The message added to a popup upon dataset publish *