From b28a4294844ed475fb3ba0546be663e72aa6a911 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 6 Apr 2017 11:33:23 -0400 Subject: [PATCH 1/8] add initial Swift support #3747 This branch was created based on on 4.5-export-harvest-swift which recently had code from ferrys:4.5-export-harvest-swift-update merged into it. This new branch was not a straight merge, however, because there was some cruft in it that we don't want in the git history. The latest from develop was merged into this new 3747-swift-and-compute-button branch and a `git reset` was performed to make a single commit with all of the changes (and only the changes) that we think we want. This single commit is being made by @pdurbin but thank you to @ferrys @anuj-rt and @landreev who did all the actual work! --- pom.xml | 24 +- .../edu/harvard/iq/dataverse/DataFile.java | 2 +- .../edu/harvard/iq/dataverse/FilePage.java | 16 +- .../api/BundleDownloadInstanceWriter.java | 2 +- .../dataverse/api/DownloadInstanceWriter.java | 2 +- .../iq/dataverse/dataaccess/DataAccess.java | 81 +++- .../dataverse/dataaccess/DataFileZipper.java | 2 +- .../dataverse/dataaccess/SwiftAccessIO.java | 404 ++++++++++++++++++ .../dataverse/ingest/IngestServiceBean.java | 9 +- .../rserve/RemoteDataFrameService.java | 2 +- .../settings/SettingsServiceBean.java | 7 +- 11 files changed, 529 insertions(+), 22 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java diff --git a/pom.xml b/pom.xml index d1bb280f86e..37af517f883 100644 --- a/pom.xml +++ b/pom.xml @@ -319,17 +319,31 @@ jacoco-maven-plugin 0.7.5.201505241946 + + + org.slf4j + slf4j-api + 1.7.7 + + + org.slf4j + slf4j-log4j12 + 1.7.7 + org.mockito mockito-core 1.10.19 - + axis axis @@ -355,6 +369,12 @@ commons-codec 1.9 + + + org.javaswift + joss + 0.9.10 + com.maxmind.geoip2 geoip2 diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index b8d6f793eef..01d34850026 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -567,7 +567,7 @@ public String getOriginalChecksumType() { } public DataFileIO getAccessObject() throws IOException { - DataFileIO dataAccess = DataAccess.createDataAccessObject(this); + DataFileIO dataAccess = DataAccess.getDataFileIO(this); if (dataAccess == null) { throw new IOException("Failed to create access object for datafile."); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 08479dfcd6f..fe5e7bfd213 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.DatasetVersionServiceBean.RetrieveDatasetVersionResponse; +import edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.datasetutility.TwoRavensHelper; @@ -616,7 +617,20 @@ public boolean isPubliclyDownloadable() { } public String getPublicDownloadUrl() { - return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); + if (System.getProperty("dataverse.files.storage-driver-id").equals("swift")) { + String fileDownloadUrl = null; + try { + SwiftAccessIO swiftIO = (SwiftAccessIO) getFile().getAccessObject(); + swiftIO.open(); + fileDownloadUrl = swiftIO.getRemoteUrl(); + logger.info("Swift url: " + fileDownloadUrl); + } catch (Exception e) { + e.printStackTrace(); + } + return fileDownloadUrl; + } else { + return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); + } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java index f4070e9b4e5..7e32d4ae231 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java @@ -56,7 +56,7 @@ public void writeTo(BundleDownloadInstance di, Class clazz, Type type, Annota if (di.getDownloadInfo() != null && di.getDownloadInfo().getDataFile() != null) { DataAccessRequest daReq = new DataAccessRequest(); DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index d41b83c476d..a81817f04f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -64,7 +64,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index c4a64bff350..82907486895 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -21,8 +21,8 @@ package edu.harvard.iq.dataverse.dataaccess; import edu.harvard.iq.dataverse.DataFile; - import java.io.IOException; +import org.javaswift.joss.model.StoredObject; /** * @@ -34,21 +34,35 @@ public DataAccess() { } - public static DataFileIO createDataAccessObject (DataFile df) throws IOException { - return createDataAccessObject (df, null); + // set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift + public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); + public static final String DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS = System.getProperty("dataverse.files.swift-endpoint-start"); + public static String swiftFileUri; + public static String swiftContainerUri; + + // The getDataFileIO() methods initialize DataFileIO objects for + // datafiles that are already saved using one of the supported Dataverse + // DataAccess IO drivers. + public static DataFileIO getDataFileIO(DataFile df) throws IOException { + return getDataFileIO(df, null); } - public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest req) throws IOException { + public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throws IOException { - if (df == null || - df.getStorageIdentifier() == null || - df.getStorageIdentifier().equals("")) { - throw new IOException ("createDataAccessObject: null or invalid datafile."); + if (df == null + || df.getStorageIdentifier() == null + || df.getStorageIdentifier().equals("")) { + throw new IOException("getDataAccessObject: null or invalid datafile."); } if (df.getStorageIdentifier().startsWith("file://") || (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) { return new FileAccessIO (df, req); + } else if (df.getStorageIdentifier().startsWith("swift://") + || df.getStorageIdentifier().startsWith(DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + return new SwiftAccessIO(df, req); + } else if (df.getStorageIdentifier().startsWith("tmp://")) { + throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet."); } // No other storage methods are supported as of now! -- 4.0.1 @@ -58,6 +72,55 @@ public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest // "storage identifier". // -- L.A. 4.0.2 - throw new IOException ("createDataAccessObject: Unsupported storage method."); + throw new IOException("getDataAccessObject: Unsupported storage method."); + } + + // createDataAccessObject() methods create a *new*, empty DataAccess objects, + // for saving new, not yet saved datafiles. + public static DataFileIO createNewDataFileIO(DataFile df, String storageTag) throws IOException { + + return createNewDataFileIO(df, storageTag, DEFAULT_STORAGE_DRIVER_IDENTIFIER); } + + public static DataFileIO createNewDataFileIO(DataFile df, String storageTag, String driverIdentifier) throws IOException { + if (df == null + || storageTag == null + || storageTag.equals("")) { + throw new IOException("getDataAccessObject: null or invalid datafile."); + } + + DataFileIO dataFileIO = null; + + df.setStorageIdentifier(storageTag); + + if (driverIdentifier.equals("file")) { + dataFileIO = new FileAccessIO(df, null); + } else if (driverIdentifier.equals("swift")) { + dataFileIO = new SwiftAccessIO(df, null); + } else { + throw new IOException("createDataAccessObject: Unsupported storage method " + driverIdentifier); + } + + dataFileIO.open(DataAccessOption.WRITE_ACCESS); + return dataFileIO; + } + + public static String getSwiftFileURI(StoredObject fileObject) throws IOException { + String fileUri; + try { + fileUri = fileObject.getPublicURL(); + } catch (Exception ex) { + ex.printStackTrace(); + throw new IOException("SwiftAccessIO: failed to get file storage location"); + } + return fileUri; + } + + public static String getSwiftContainerURI(StoredObject fileObject) throws IOException { + String containerUri; + containerUri = getSwiftFileURI(fileObject); + containerUri = containerUri.substring(0, containerUri.lastIndexOf('/')); + return containerUri; + } + } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java index e75f326c3fa..d9022d87654 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java @@ -221,7 +221,7 @@ public long addFileToZipStream(DataFile dataFile) throws IOException { boolean createManifest = fileManifest != null; DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java new file mode 100644 index 00000000000..adc6518c819 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -0,0 +1,404 @@ +package edu.harvard.iq.dataverse.dataaccess; + +import edu.harvard.iq.dataverse.DataFile; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channel; +import java.nio.channels.Channels; +import java.nio.file.Path; +import java.util.Properties; +import java.util.logging.Logger; +import org.javaswift.joss.client.factory.AccountFactory; +import static org.javaswift.joss.client.factory.AuthenticationMethod.BASIC; +import org.javaswift.joss.model.Account; +import org.javaswift.joss.model.Container; +import org.javaswift.joss.model.StoredObject; + +/** + * + * @author leonid andreev + */ +/* + Experimental Swift driver, implemented as part of the Dataverse - Mass Open Cloud + collaboration. + Read-only access, for now. + */ +public class SwiftAccessIO extends DataFileIO { + + private String swiftFolderPath; + private String swiftFileName = null; + + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO"); + + public SwiftAccessIO() throws IOException { + this(null); + } + + public SwiftAccessIO(DataFile dataFile) throws IOException { + this(dataFile, null); + + } + + public SwiftAccessIO(DataFile dataFile, DataAccessRequest req) throws IOException { + + super(dataFile, req); + + this.setIsLocalFile(false); + } + + private boolean isReadAccess = false; + private boolean isWriteAccess = false; + private Properties swiftProperties = null; + private Account account = null; + private StoredObject swiftFileObject = null; + + @Override + public boolean canRead() { + return isReadAccess; + } + + @Override + public boolean canWrite() { + return isWriteAccess; + } + + @Override + public void open(DataAccessOption... options) throws IOException { + + DataFile dataFile = this.getDataFile(); + DataAccessRequest req = this.getRequest(); + + if (req != null && req.getParameter("noVarHeader") != null) { + this.setNoVarHeader(true); + } + + if (isWriteAccessRequested(options)) { + isWriteAccess = true; + isReadAccess = false; + } else { + isWriteAccess = false; + isReadAccess = true; + } + + if (this.getDataFile().getStorageIdentifier() == null || "".equals(this.getDataFile().getStorageIdentifier())) { + throw new IOException("Data Access: No local storage identifier defined for this datafile."); + } + + if (isReadAccess) { + InputStream fin = openSwiftFileAsInputStream(); + + if (fin == null) { + throw new IOException("Failed to open Swift file " + getStorageLocation()); + } + + this.setInputStream(fin); + setChannel(Channels.newChannel(fin)); + + } else if (isWriteAccess) { + swiftFileObject = initializeSwiftFileObject(true); + } + + this.setMimeType(dataFile.getContentType()); + try { + this.setFileName(dataFile.getFileMetadata().getLabel()); + } catch (Exception ex) { + this.setFileName("unknown"); + } + + // This "status" is a leftover from 3.6; we don't have a use for it + // in 4.0 yet; and we may not need it at all. + // -- L.A. 4.0.2 + this.setStatus(200); + } + + // this is a Swift-specific override of the convenience method provided in the + // DataFileIO for copying a local Path (for ex., a temp file, into this DataAccess location): + @Override + public void copyPath(Path fileSystemPath) throws IOException { + long newFileSize = -1; + Properties p = getSwiftProperties(); + String swiftEndPoint = p.getProperty("swift.default.endpoint"); + String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint); + + if (swiftFileObject == null || !this.canWrite()) { + open(DataAccessOption.WRITE_ACCESS); + } + + File inputFile = null; + + try { + inputFile = fileSystemPath.toFile(); + + //@author Anuj Thakur + swiftFileObject.uploadObject(inputFile); + //After the files object is uploaded the identifier is changed. + logger.info(this.swiftFileName + " " + this.swiftFolderPath); + this.getDataFile().setStorageIdentifier(swiftDirectory + "/" + this.swiftFolderPath + "/" + this.swiftFileName); + + newFileSize = inputFile.length(); + + } catch (Exception ioex) { + String failureMsg = ioex.getMessage(); + if (failureMsg == null) { + failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject"; + } + + throw new IOException(failureMsg); + } + + // if it has uploaded successfully, we can reset the size + // of the object: + setSize(newFileSize); + + } + + @Override + public void delete() throws IOException { + //throw new IOException("SwiftAccessIO: delete() not yet implemented in this storage driver."); + if (swiftFileObject == null) { + try { + swiftFileObject = initializeSwiftFileObject(false); + } catch (IOException ioex) { + swiftFileObject = null; + } + } + + if (swiftFileObject != null) { + swiftFileObject.delete(); + } + } + + @Override + public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException { + + throw new IOException("SwiftAccessIO: openAuxChannel() not yet implemented in this storage driver."); + } + + @Override + public boolean isAuxObjectCached(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: isAuxObjectCached() not yet implemented in this storage driver."); + } + + @Override + public long getAuxObjectSize(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: getAuxObjectSize() not yet implemented in this storage driver."); + } + + @Override + public void backupAsAux(String auxItemTag) throws IOException { + throw new IOException("SwiftAccessIO: backupAsAux() not yet implemented in this storage driver."); + } + + @Override + public String getStorageLocation() { + // What should this be, for a Swift file? + // A Swift URL? + // Or a Swift URL with an authenticated Auth token? + + return null; + } + + @Override + public Path getFileSystemPath() throws IOException { + throw new IOException("SwiftAccessIO: this is a remote AccessIO object, it has no local filesystem path associated with it."); + } + + // Auxilary helper methods, Swift-specific: + private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOException { + String storageIdentifier = this.getDataFile().getStorageIdentifier(); + + String swiftEndPoint = null; + String swiftContainer = null; + + // + // if (storageIdentifier.startsWith("swift://")) { + if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + // This is a call on an already existing swift object. + + //String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); + //The Storage identifier now is the Object store service endpoint + //followed by the endpoint named container + String[] swiftStorageTokens = storageIdentifier.substring(46).split("/", 2); + + if (swiftStorageTokens.length != 2) { + // bad storage identifier + throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); + } + + Properties p = getSwiftProperties(); + swiftEndPoint = p.getProperty("swift.default.endpoint"); + + swiftContainer = swiftStorageTokens[0]; + swiftFileName = swiftStorageTokens[1]; + + if (swiftEndPoint == null || swiftContainer == null || swiftFileName == null + || "".equals(swiftEndPoint) || "".equals(swiftContainer) || "".equals(swiftFileName)) { + // all of these things need to be specified, for this to be a valid Swift location + // identifier. + throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); + } + } else if (this.isReadAccess) { + // An attempt to call Swift driver, in a Read mode on a non-swift stored datafile + // object! + throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); + } else if (this.isWriteAccess) { + Properties p = getSwiftProperties(); + swiftEndPoint = p.getProperty("swift.default.endpoint"); + swiftFolderPath = this.getDataFile().getOwner().getDisplayName(); + //swiftFolderPath = this.getDataFile().getOwner().getIdentifier(); /* TODO: ? */ + swiftFileName = storageIdentifier; + //swiftFileName = this.getDataFile().getDisplayName(); + //Storage Identifier is now updated after the object is uploaded on Swift. + // this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftContainer+":"+swiftFileName); + } else { + throw new IOException("SwiftAccessIO: unknown access mode."); + } + // Authenticate with Swift: + + account = authenticateWithSwift(swiftEndPoint); + + /* + The containers created is swiftEndPoint concatenated with the swiftContainer + property. Creating container with certain names throws 'Unable to create + container' error on Openstack. + Any datafile with http://rdgw storage identifier i.e present on Object + store service endpoint already only needs to look-up for container using + just swiftContainer which is the concatenated name. + In future, a container for the endpoint can be created and for every + other swiftContainer Object Store pseudo-folder can be created, which is + not provide by the joss Java swift library as of yet. + */ + Container dataContainer; + + if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + dataContainer = account.getContainer(swiftContainer); + } else { + dataContainer = account.getContainer(swiftFolderPath); //changed from swiftendpoint + } + + if (!dataContainer.exists()) { + if (writeAccess) { + dataContainer.create(); + //dataContainer.makePublic(); + } else { + // This is a fatal condition - it has to exist, if we were to + // read an existing object! + throw new IOException("SwiftAccessIO: container " + swiftContainer + " does not exist."); + } + } + + StoredObject fileObject = dataContainer.getObject(swiftFileName); + //file download url for public files + DataAccess.swiftFileUri = DataAccess.getSwiftFileURI(fileObject); + setRemoteUrl(DataAccess.getSwiftFileURI(fileObject)); + + logger.info(DataAccess.swiftFileUri + " success"); + //shows contents of container for public containers + DataAccess.swiftContainerUri = DataAccess.getSwiftContainerURI(fileObject); + logger.info(DataAccess.swiftContainerUri + " success"); + + if (!writeAccess && !fileObject.exists()) { + throw new IOException("SwiftAccessIO: File object " + swiftFileName + " does not exist (Dataverse datafile id: " + this.getDataFile().getId()); + } + + return fileObject; + } + + private InputStream openSwiftFileAsInputStream() throws IOException { + InputStream in = null; + + swiftFileObject = initializeSwiftFileObject(false); + + in = swiftFileObject.downloadObjectAsInputStream(); + this.setSize(swiftFileObject.getContentLength()); + + return in; + } + + private Properties getSwiftProperties() throws IOException { + if (swiftProperties == null) { + String domainRoot = System.getProperties().getProperty("com.sun.aas.instanceRoot"); + String swiftPropertiesFile = domainRoot + File.separator + "config" + File.separator + "swift.properties"; + swiftProperties = new Properties(); + swiftProperties.load(new FileInputStream(new File(swiftPropertiesFile))); + } + + return swiftProperties; + } + + Account authenticateWithSwift(String swiftEndPoint) throws IOException { + + Properties p = getSwiftProperties(); + + // (this will throw an IOException, if the swift properties file + // is missing or corrupted) + String swiftEndPointAuthUrl = p.getProperty("swift.auth_url." + swiftEndPoint); + String swiftEndPointUsername = p.getProperty("swift.username." + swiftEndPoint); + String swiftEndPointSecretKey = p.getProperty("swift.password." + swiftEndPoint); + String swiftEndPointTenantName = p.getProperty("swift.tenant." + swiftEndPoint); + String swiftEndPointAuthMethod = p.getProperty("swift.auth_type." + swiftEndPoint); + + if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null + || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) { + // again, all of these things need to be defined, for this Swift endpoint to be + // accessible. + throw new IOException("SwiftAccessIO: no configuration available for endpoint " + swiftEndPoint); + } + + // Authenticate: + Account account = null; + + /* + This try { } now authenticates using either the KEYSTONE mechanism which uses + the tenant name in addition to the Username Password and AuthUrl OR the BASIC method + Also, the AuthUrl is now the identity service endpoint of MOC Openstack + environment instead of the Object store service endpoint. + */ + // Keystone vs. Basic + try { + if (swiftEndPointAuthMethod.equals("keystone")) { + account = new AccountFactory() + .setTenantName(swiftEndPointTenantName) + .setUsername(swiftEndPointUsername) + .setPassword(swiftEndPointSecretKey) + .setAuthUrl(swiftEndPointAuthUrl) + .createAccount(); + } else { // assume BASIC + account = new AccountFactory() + .setUsername(swiftEndPointUsername) + .setPassword(swiftEndPointSecretKey) + .setAuthUrl(swiftEndPointAuthUrl) + .setAuthenticationMethod(BASIC) + .createAccount(); + } + + } catch (Exception ex) { + ex.printStackTrace(); + throw new IOException("SwiftAccessIO: failed to authenticate " + swiftEndPointAuthMethod + " for the end point " + swiftEndPoint); + } + + return account; + } + + private boolean isWriteAccessRequested(DataAccessOption... options) throws IOException { + + for (DataAccessOption option : options) { + // In the future we may need to be able to open read-write + // Channels; no support, or use case for that as of now. + + if (option == DataAccessOption.READ_ACCESS) { + return false; + } + + if (option == DataAccessOption.WRITE_ACCESS) { + return true; + } + } + + // By default, we open the file in read mode: + return false; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index f9ab137a603..be538442d22 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -37,6 +37,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.DataFileIO; import edu.harvard.iq.dataverse.dataaccess.FileAccessIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -267,12 +268,14 @@ public void addFiles (DatasetVersion version, List newFiles) { try { - DataFileIO dataAccess = dataFile.getAccessObject(); + logger.info("Attempting to create a new DataFileIO object for " + storageId); + DataFileIO dataAccess = DataAccess.createNewDataFileIO(dataFile, storageId); if (dataAccess.isLocalFile()) { localFile = true; } - + + logger.info("Successfully created a new DataFileIO object."); /* This commented-out code demonstrates how to copy bytes from a local InputStream (or a readChannel) into the @@ -311,7 +314,7 @@ from a local InputStream (or a readChannel) into the savedSuccess = true; } catch (IOException ioex) { - logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier()); + logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); } finally { if (readChannel != null) {try{readChannel.close();}catch(IOException e){}} if (writeChannel != null) {try{writeChannel.close();}catch(IOException e){}} diff --git a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java index 791b376fb1e..df81d0328aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java +++ b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java @@ -564,7 +564,7 @@ public File runDataPreprocessing(DataFile dataFile) { // send the tabular data file to the Rserve side: DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); + DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); if (accessObject == null) { return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 0211fedf0e7..fbe67d4c6e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -222,8 +222,11 @@ public enum Key { Default is false; */ TwoRavensTabularView, - - + /** + * Whether to save data files locally in files or store them in swift + * storage, default is file. + */ + Driver, /** The message added to a popup upon dataset publish * From 6f1e12cb51b61a13a17b60f3543ddbad690d2fee Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 7 Apr 2017 13:45:40 -0400 Subject: [PATCH 2/8] quick fix to prevent a NullPointerException #3747 --- .../java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 82907486895..8367fe587be 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -93,6 +93,10 @@ public static DataFileIO createNewDataFileIO(DataFile df, String storageTag, Str df.setStorageIdentifier(storageTag); + if (driverIdentifier == null) { + driverIdentifier = "file"; + } + if (driverIdentifier.equals("file")) { dataFileIO = new FileAccessIO(df, null); } else if (driverIdentifier.equals("swift")) { From f74fe4125c91a6f3f8fef0be7f01e43a9bc6cab1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 7 Apr 2017 14:28:09 -0400 Subject: [PATCH 3/8] catch ClassCastException and bail out #3747 --- .../harvard/iq/dataverse/dataaccess/ImageThumbConverter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index 419b6babfc4..c4ce2dfc532 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -152,6 +152,9 @@ public static String getImageThumbAsBase64(DataFile file, int size) { try { fileAccess = (FileAccessIO) file.getAccessObject(); + } catch (ClassCastException ex) { + logger.warning("Unable to cast file id " + file.getId() + " from DataFileIO to FileAccessIO."); + return null; } catch (IOException ex) { // too bad - but not fatal logger.warning("getImageThumbAsBase64: Failed to obtain FileAccess object for DataFile id " + file.getId()); From dc3089387ad36c390cc32c18ae460129f226014a Mon Sep 17 00:00:00 2001 From: Sarah F Date: Mon, 10 Apr 2017 20:14:32 +0000 Subject: [PATCH 4/8] Updated storage identifier and swift container ID --- .../iq/dataverse/dataaccess/DataAccess.java | 4 +- .../dataverse/dataaccess/SwiftAccessIO.java | 83 +++++++++++++------ 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 8367fe587be..e0b3d6c649e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -36,7 +36,6 @@ public DataAccess() { // set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); - public static final String DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS = System.getProperty("dataverse.files.swift-endpoint-start"); public static String swiftFileUri; public static String swiftContainerUri; @@ -58,8 +57,7 @@ public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throw if (df.getStorageIdentifier().startsWith("file://") || (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) { return new FileAccessIO (df, req); - } else if (df.getStorageIdentifier().startsWith("swift://") - || df.getStorageIdentifier().startsWith(DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + } else if (df.getStorageIdentifier().startsWith("swift://")){ return new SwiftAccessIO(df, req); } else if (df.getStorageIdentifier().startsWith("tmp://")) { throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet."); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index adc6518c819..7c8385adf5d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -115,12 +115,10 @@ public void open(DataAccessOption... options) throws IOException { // this is a Swift-specific override of the convenience method provided in the // DataFileIO for copying a local Path (for ex., a temp file, into this DataAccess location): - @Override + + @Override public void copyPath(Path fileSystemPath) throws IOException { long newFileSize = -1; - Properties p = getSwiftProperties(); - String swiftEndPoint = p.getProperty("swift.default.endpoint"); - String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint); if (swiftFileObject == null || !this.canWrite()) { open(DataAccessOption.WRITE_ACCESS); @@ -131,11 +129,7 @@ public void copyPath(Path fileSystemPath) throws IOException { try { inputFile = fileSystemPath.toFile(); - //@author Anuj Thakur swiftFileObject.uploadObject(inputFile); - //After the files object is uploaded the identifier is changed. - logger.info(this.swiftFileName + " " + this.swiftFolderPath); - this.getDataFile().setStorageIdentifier(swiftDirectory + "/" + this.swiftFolderPath + "/" + this.swiftFileName); newFileSize = inputFile.length(); @@ -153,6 +147,44 @@ public void copyPath(Path fileSystemPath) throws IOException { setSize(newFileSize); } + // @Override + // public void copyPath(Path fileSystemPath) throws IOException { + // long newFileSize = -1; + // Properties p = getSwiftProperties(); + // String swiftEndPoint = p.getProperty("swift.default.endpoint"); + // String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint); + + // if (swiftFileObject == null || !this.canWrite()) { + // open(DataAccessOption.WRITE_ACCESS); + // } + + // File inputFile = null; + + // try { + // inputFile = fileSystemPath.toFile(); + + // //@author Anuj Thakur + // swiftFileObject.uploadObject(inputFile); + // //After the files object is uploaded the identifier is changed. + // logger.info(this.swiftFileName + " " + this.swiftFolderPath); + // this.getDataFile().setStorageIdentifier( + this.swiftFolderPath + "/" + this.swiftFileName); + + // newFileSize = inputFile.length(); + + // } catch (Exception ioex) { + // String failureMsg = ioex.getMessage(); + // if (failureMsg == null) { + // failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject"; + // } + + // throw new IOException(failureMsg); + // } + + // // if it has uploaded successfully, we can reset the size + // // of the object: + // setSize(newFileSize); + + // } @Override public void delete() throws IOException { @@ -212,26 +244,20 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc String swiftEndPoint = null; String swiftContainer = null; - // - // if (storageIdentifier.startsWith("swift://")) { - if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + + if (storageIdentifier.startsWith("swift://")) { // This is a call on an already existing swift object. - - //String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); - //The Storage identifier now is the Object store service endpoint - //followed by the endpoint named container - String[] swiftStorageTokens = storageIdentifier.substring(46).split("/", 2); - - if (swiftStorageTokens.length != 2) { + + String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); + + if (swiftStorageTokens.length != 3) { // bad storage identifier throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); } - Properties p = getSwiftProperties(); - swiftEndPoint = p.getProperty("swift.default.endpoint"); - - swiftContainer = swiftStorageTokens[0]; - swiftFileName = swiftStorageTokens[1]; + swiftEndPoint = swiftStorageTokens[0]; + swiftContainer = swiftStorageTokens[1]; + swiftFileName = swiftStorageTokens[2]; if (swiftEndPoint == null || swiftContainer == null || swiftFileName == null || "".equals(swiftEndPoint) || "".equals(swiftContainer) || "".equals(swiftFileName)) { @@ -246,12 +272,15 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc } else if (this.isWriteAccess) { Properties p = getSwiftProperties(); swiftEndPoint = p.getProperty("swift.default.endpoint"); - swiftFolderPath = this.getDataFile().getOwner().getDisplayName(); - //swiftFolderPath = this.getDataFile().getOwner().getIdentifier(); /* TODO: ? */ + + //swiftFolderPath = this.getDataFile().getOwner().getDisplayName(); + String swiftFolderPathSeparator = "_"; + swiftFolderPath = this.getDataFile().getOwner().getAuthority().replace(this.getDataFile().getOwner().getDoiSeparator(), swiftFolderPathSeparator) + + swiftFolderPathSeparator + this.getDataFile().getOwner().getIdentifier(); swiftFileName = storageIdentifier; //swiftFileName = this.getDataFile().getDisplayName(); //Storage Identifier is now updated after the object is uploaded on Swift. - // this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftContainer+":"+swiftFileName); + this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftFolderPath+":"+swiftFileName); } else { throw new IOException("SwiftAccessIO: unknown access mode."); } @@ -272,7 +301,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc */ Container dataContainer; - if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) { + if (storageIdentifier.startsWith("swift://")) { dataContainer = account.getContainer(swiftContainer); } else { dataContainer = account.getContainer(swiftFolderPath); //changed from swiftendpoint From f46c6675a3726ec93f1bb126ba6b4993357f049a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 20 Apr 2017 09:57:35 -0400 Subject: [PATCH 5/8] add "Data Not Published" popup to file page on "Map Data" click #3760 --- src/main/webapp/file-download-button-fragment.xhtml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index b7788f3fd62..bd5937a3551 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -275,4 +275,14 @@ #{fileMetadata.dataFile.fileAccessRequesters.contains(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']} + +

+ #{bundle['file.mapData.unpublished.message']} +

+
+ +
+ From c88f7bd27bb51b7a27a9d898d343742d55602c12 Mon Sep 17 00:00:00 2001 From: Sarah F Date: Fri, 21 Apr 2017 16:31:35 +0000 Subject: [PATCH 6/8] Updated config docs and container names --- .../source/installation/config.rst | 28 +++++++++++++++++++ .../iq/dataverse/dataaccess/DataAccess.java | 8 +----- .../dataverse/dataaccess/SwiftAccessIO.java | 7 ++--- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 430a5fb9ec9..9fd8511c071 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -149,6 +149,29 @@ Enabling a second authentication provider will result in the Log In page showing - ``:AllowSignUp`` is set to "false" per the :doc:`config` section to prevent users from creating local accounts via the web interface. Please note that local accounts can also be created via API, and the way to prevent this is to block the ``builtin-users`` endpoint or scramble (or remove) the ``BuiltinUsers.KEY`` database setting per the :doc:`config` section. - The "builtin" authentication provider has been disabled. Note that disabling the builting auth provider means that the API endpoint for converting an account from a remote auth provider will not work. This is the main reason why https://github.com/IQSS/dataverse/issues/2974 is still open. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to builtin. Then the user initiates a conversion from builtin to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. +Swift Installation +------------------ + +On every generic Dataverse installation, datafiles are stored in local storage. You can opt for a experimental dataverse installation with a Swift Object Storage backend instead. Each dataset that you create is saved as a container. Each datafile is saved as a file within the container. + +In order to configure a Swift installation, there are two steps you need to complete before running the installer: + +First, you need to add a file called swift.properties in ``/glassfish4/glassfish/domains/domain1/config``. It needs to be configured as follows: + +.. code-block:: none + swift.default.endpoint=endpoint1 + swift.auth_url.endpoint1=your-auth-url + swift.tenant.endpoint1=your-tenant-name + swift.username.endpoint1=your-username + swift.password.endpoint1=your-password + swift.auth_type.endpoint1=your-authentication-type + swift.swift_endpoint.endpoint1=your-swift-endpoint + +where authentication type can either be "keystone" (without the quotes) or it will assumed to be basic. Additionally, authentication url should be your keystone authentication url which includes the tokens. For example, https://dataverse.org:35357/v2.0/tokens. + +Second, you need to update the JVM option ``dataverse.files.storage-driver-id``. You can do this by running the delete command ``./asadmin $ASADMIN_OPTS delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"`` and then the create command ``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.storage-driver-id=swift"``. + + JVM Options ----------- @@ -188,6 +211,11 @@ dataverse.files.directory This is how you configure the path to which files uploaded by users are stored. +dataverse.files.storage-driver-id ++++++++++++++++++++++++++++++++++ + +Initialized to file, this is what you would change if you were interested in a Swift backend for your installation. See above in the Swift Installation section. + dataverse.auth.password-reset-timeout-in-minutes ++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index e0b3d6c649e..f8483dfa9ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -37,7 +37,6 @@ public DataAccess() { // set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); public static String swiftFileUri; - public static String swiftContainerUri; // The getDataFileIO() methods initialize DataFileIO objects for // datafiles that are already saved using one of the supported Dataverse @@ -118,11 +117,6 @@ public static String getSwiftFileURI(StoredObject fileObject) throws IOException return fileUri; } - public static String getSwiftContainerURI(StoredObject fileObject) throws IOException { - String containerUri; - containerUri = getSwiftFileURI(fileObject); - containerUri = containerUri.substring(0, containerUri.lastIndexOf('/')); - return containerUri; - } + } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index 7c8385adf5d..951e0c21663 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -275,7 +275,9 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc //swiftFolderPath = this.getDataFile().getOwner().getDisplayName(); String swiftFolderPathSeparator = "_"; - swiftFolderPath = this.getDataFile().getOwner().getAuthority().replace(this.getDataFile().getOwner().getDoiSeparator(), swiftFolderPathSeparator) + + String authorityNoSlashes = this.getDataFile().getOwner().getAuthority().replace(this.getDataFile().getOwner().getDoiSeparator(), swiftFolderPathSeparator); + swiftFolderPath = this.getDataFile().getOwner().getProtocol() + swiftFolderPathSeparator + + authorityNoSlashes.replace(".", swiftFolderPathSeparator) + swiftFolderPathSeparator + this.getDataFile().getOwner().getIdentifier(); swiftFileName = storageIdentifier; //swiftFileName = this.getDataFile().getDisplayName(); @@ -324,9 +326,6 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc setRemoteUrl(DataAccess.getSwiftFileURI(fileObject)); logger.info(DataAccess.swiftFileUri + " success"); - //shows contents of container for public containers - DataAccess.swiftContainerUri = DataAccess.getSwiftContainerURI(fileObject); - logger.info(DataAccess.swiftContainerUri + " success"); if (!writeAccess && !fileObject.exists()) { throw new IOException("SwiftAccessIO: File object " + swiftFileName + " does not exist (Dataverse datafile id: " + this.getDataFile().getId()); From 6f1f5ec3f85658c054573de181905fd598670da3 Mon Sep 17 00:00:00 2001 From: Sarah Ferry Date: Fri, 21 Apr 2017 12:48:17 -0400 Subject: [PATCH 7/8] Revert "Updated config docs and container names" --- .../source/installation/config.rst | 28 -- pom.xml | 24 +- .../edu/harvard/iq/dataverse/DataFile.java | 2 +- .../edu/harvard/iq/dataverse/FilePage.java | 16 +- .../api/BundleDownloadInstanceWriter.java | 2 +- .../dataverse/api/DownloadInstanceWriter.java | 2 +- .../iq/dataverse/dataaccess/DataAccess.java | 77 +--- .../dataverse/dataaccess/DataFileZipper.java | 2 +- .../dataaccess/ImageThumbConverter.java | 3 - .../dataverse/dataaccess/SwiftAccessIO.java | 432 ------------------ .../dataverse/ingest/IngestServiceBean.java | 9 +- .../rserve/RemoteDataFrameService.java | 2 +- .../settings/SettingsServiceBean.java | 7 +- 13 files changed, 22 insertions(+), 584 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 62acc544e50..7c95b3453fd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -149,29 +149,6 @@ Enabling a second authentication provider will result in the Log In page showing - ``:AllowSignUp`` is set to "false" per the :doc:`config` section to prevent users from creating local accounts via the web interface. Please note that local accounts can also be created via API, and the way to prevent this is to block the ``builtin-users`` endpoint or scramble (or remove) the ``BuiltinUsers.KEY`` database setting per the :doc:`config` section. - The "builtin" authentication provider has been disabled. Note that disabling the builting auth provider means that the API endpoint for converting an account from a remote auth provider will not work. This is the main reason why https://github.com/IQSS/dataverse/issues/2974 is still open. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to builtin. Then the user initiates a conversion from builtin to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. -Swift Installation ------------------- - -On every generic Dataverse installation, datafiles are stored in local storage. You can opt for a experimental dataverse installation with a Swift Object Storage backend instead. Each dataset that you create is saved as a container. Each datafile is saved as a file within the container. - -In order to configure a Swift installation, there are two steps you need to complete before running the installer: - -First, you need to add a file called swift.properties in ``/glassfish4/glassfish/domains/domain1/config``. It needs to be configured as follows: - -.. code-block:: none - swift.default.endpoint=endpoint1 - swift.auth_url.endpoint1=your-auth-url - swift.tenant.endpoint1=your-tenant-name - swift.username.endpoint1=your-username - swift.password.endpoint1=your-password - swift.auth_type.endpoint1=your-authentication-type - swift.swift_endpoint.endpoint1=your-swift-endpoint - -where authentication type can either be "keystone" (without the quotes) or it will assumed to be basic. Additionally, authentication url should be your keystone authentication url which includes the tokens. For example, https://dataverse.org:35357/v2.0/tokens. - -Second, you need to update the JVM option ``dataverse.files.storage-driver-id``. You can do this by running the delete command ``./asadmin $ASADMIN_OPTS delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"`` and then the create command ``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.storage-driver-id=swift"``. - - JVM Options ----------- @@ -211,11 +188,6 @@ dataverse.files.directory This is how you configure the path to which files uploaded by users are stored. -dataverse.files.storage-driver-id -+++++++++++++++++++++++++++++++++ - -Initialized to file, this is what you would change if you were interested in a Swift backend for your installation. See above in the Swift Installation section. - dataverse.auth.password-reset-timeout-in-minutes ++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/pom.xml b/pom.xml index 37af517f883..d1bb280f86e 100644 --- a/pom.xml +++ b/pom.xml @@ -319,31 +319,17 @@ jacoco-maven-plugin 0.7.5.201505241946 - - - org.slf4j - slf4j-api - 1.7.7 - - - org.slf4j - slf4j-log4j12 - 1.7.7 - org.mockito mockito-core 1.10.19 - + axis axis @@ -369,12 +355,6 @@ commons-codec 1.9 - - - org.javaswift - joss - 0.9.10 - com.maxmind.geoip2 geoip2 diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 6b415542c99..dfba4ca684d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -568,7 +568,7 @@ public String getOriginalChecksumType() { } public DataFileIO getAccessObject() throws IOException { - DataFileIO dataAccess = DataAccess.getDataFileIO(this); + DataFileIO dataAccess = DataAccess.createDataAccessObject(this); if (dataAccess == null) { throw new IOException("Failed to create access object for datafile."); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index fe5e7bfd213..08479dfcd6f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -6,7 +6,6 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.DatasetVersionServiceBean.RetrieveDatasetVersionResponse; -import edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.datasetutility.TwoRavensHelper; @@ -617,20 +616,7 @@ public boolean isPubliclyDownloadable() { } public String getPublicDownloadUrl() { - if (System.getProperty("dataverse.files.storage-driver-id").equals("swift")) { - String fileDownloadUrl = null; - try { - SwiftAccessIO swiftIO = (SwiftAccessIO) getFile().getAccessObject(); - swiftIO.open(); - fileDownloadUrl = swiftIO.getRemoteUrl(); - logger.info("Swift url: " + fileDownloadUrl); - } catch (Exception e) { - e.printStackTrace(); - } - return fileDownloadUrl; - } else { - return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); - } + return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), fileId); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java index 7e32d4ae231..f4070e9b4e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BundleDownloadInstanceWriter.java @@ -56,7 +56,7 @@ public void writeTo(BundleDownloadInstance di, Class clazz, Type type, Annota if (di.getDownloadInfo() != null && di.getDownloadInfo().getDataFile() != null) { DataAccessRequest daReq = new DataAccessRequest(); DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); + DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 2d5008d54f2..c611cd6e796 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -64,7 +64,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] DataFile sf = di.getDownloadInfo().getDataFile(); - DataFileIO accessObject = DataAccess.getDataFileIO(sf, daReq); + DataFileIO accessObject = DataAccess.createDataAccessObject(sf, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index f8483dfa9ee..c4a64bff350 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -21,8 +21,8 @@ package edu.harvard.iq.dataverse.dataaccess; import edu.harvard.iq.dataverse.DataFile; + import java.io.IOException; -import org.javaswift.joss.model.StoredObject; /** * @@ -34,32 +34,21 @@ public DataAccess() { } - // set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift - public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); - public static String swiftFileUri; - - // The getDataFileIO() methods initialize DataFileIO objects for - // datafiles that are already saved using one of the supported Dataverse - // DataAccess IO drivers. - public static DataFileIO getDataFileIO(DataFile df) throws IOException { - return getDataFileIO(df, null); + public static DataFileIO createDataAccessObject (DataFile df) throws IOException { + return createDataAccessObject (df, null); } - public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throws IOException { + public static DataFileIO createDataAccessObject (DataFile df, DataAccessRequest req) throws IOException { - if (df == null - || df.getStorageIdentifier() == null - || df.getStorageIdentifier().equals("")) { - throw new IOException("getDataAccessObject: null or invalid datafile."); + if (df == null || + df.getStorageIdentifier() == null || + df.getStorageIdentifier().equals("")) { + throw new IOException ("createDataAccessObject: null or invalid datafile."); } if (df.getStorageIdentifier().startsWith("file://") || (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) { return new FileAccessIO (df, req); - } else if (df.getStorageIdentifier().startsWith("swift://")){ - return new SwiftAccessIO(df, req); - } else if (df.getStorageIdentifier().startsWith("tmp://")) { - throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet."); } // No other storage methods are supported as of now! -- 4.0.1 @@ -69,54 +58,6 @@ public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throw // "storage identifier". // -- L.A. 4.0.2 - throw new IOException("getDataAccessObject: Unsupported storage method."); - } - - // createDataAccessObject() methods create a *new*, empty DataAccess objects, - // for saving new, not yet saved datafiles. - public static DataFileIO createNewDataFileIO(DataFile df, String storageTag) throws IOException { - - return createNewDataFileIO(df, storageTag, DEFAULT_STORAGE_DRIVER_IDENTIFIER); + throw new IOException ("createDataAccessObject: Unsupported storage method."); } - - public static DataFileIO createNewDataFileIO(DataFile df, String storageTag, String driverIdentifier) throws IOException { - if (df == null - || storageTag == null - || storageTag.equals("")) { - throw new IOException("getDataAccessObject: null or invalid datafile."); - } - - DataFileIO dataFileIO = null; - - df.setStorageIdentifier(storageTag); - - if (driverIdentifier == null) { - driverIdentifier = "file"; - } - - if (driverIdentifier.equals("file")) { - dataFileIO = new FileAccessIO(df, null); - } else if (driverIdentifier.equals("swift")) { - dataFileIO = new SwiftAccessIO(df, null); - } else { - throw new IOException("createDataAccessObject: Unsupported storage method " + driverIdentifier); - } - - dataFileIO.open(DataAccessOption.WRITE_ACCESS); - return dataFileIO; - } - - public static String getSwiftFileURI(StoredObject fileObject) throws IOException { - String fileUri; - try { - fileUri = fileObject.getPublicURL(); - } catch (Exception ex) { - ex.printStackTrace(); - throw new IOException("SwiftAccessIO: failed to get file storage location"); - } - return fileUri; - } - - - } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java index d9022d87654..e75f326c3fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java @@ -221,7 +221,7 @@ public long addFileToZipStream(DataFile dataFile) throws IOException { boolean createManifest = fileManifest != null; DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); + DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); if (accessObject != null) { accessObject.open(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index c4ce2dfc532..419b6babfc4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -152,9 +152,6 @@ public static String getImageThumbAsBase64(DataFile file, int size) { try { fileAccess = (FileAccessIO) file.getAccessObject(); - } catch (ClassCastException ex) { - logger.warning("Unable to cast file id " + file.getId() + " from DataFileIO to FileAccessIO."); - return null; } catch (IOException ex) { // too bad - but not fatal logger.warning("getImageThumbAsBase64: Failed to obtain FileAccess object for DataFile id " + file.getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java deleted file mode 100644 index 951e0c21663..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ /dev/null @@ -1,432 +0,0 @@ -package edu.harvard.iq.dataverse.dataaccess; - -import edu.harvard.iq.dataverse.DataFile; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channel; -import java.nio.channels.Channels; -import java.nio.file.Path; -import java.util.Properties; -import java.util.logging.Logger; -import org.javaswift.joss.client.factory.AccountFactory; -import static org.javaswift.joss.client.factory.AuthenticationMethod.BASIC; -import org.javaswift.joss.model.Account; -import org.javaswift.joss.model.Container; -import org.javaswift.joss.model.StoredObject; - -/** - * - * @author leonid andreev - */ -/* - Experimental Swift driver, implemented as part of the Dataverse - Mass Open Cloud - collaboration. - Read-only access, for now. - */ -public class SwiftAccessIO extends DataFileIO { - - private String swiftFolderPath; - private String swiftFileName = null; - - private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO"); - - public SwiftAccessIO() throws IOException { - this(null); - } - - public SwiftAccessIO(DataFile dataFile) throws IOException { - this(dataFile, null); - - } - - public SwiftAccessIO(DataFile dataFile, DataAccessRequest req) throws IOException { - - super(dataFile, req); - - this.setIsLocalFile(false); - } - - private boolean isReadAccess = false; - private boolean isWriteAccess = false; - private Properties swiftProperties = null; - private Account account = null; - private StoredObject swiftFileObject = null; - - @Override - public boolean canRead() { - return isReadAccess; - } - - @Override - public boolean canWrite() { - return isWriteAccess; - } - - @Override - public void open(DataAccessOption... options) throws IOException { - - DataFile dataFile = this.getDataFile(); - DataAccessRequest req = this.getRequest(); - - if (req != null && req.getParameter("noVarHeader") != null) { - this.setNoVarHeader(true); - } - - if (isWriteAccessRequested(options)) { - isWriteAccess = true; - isReadAccess = false; - } else { - isWriteAccess = false; - isReadAccess = true; - } - - if (this.getDataFile().getStorageIdentifier() == null || "".equals(this.getDataFile().getStorageIdentifier())) { - throw new IOException("Data Access: No local storage identifier defined for this datafile."); - } - - if (isReadAccess) { - InputStream fin = openSwiftFileAsInputStream(); - - if (fin == null) { - throw new IOException("Failed to open Swift file " + getStorageLocation()); - } - - this.setInputStream(fin); - setChannel(Channels.newChannel(fin)); - - } else if (isWriteAccess) { - swiftFileObject = initializeSwiftFileObject(true); - } - - this.setMimeType(dataFile.getContentType()); - try { - this.setFileName(dataFile.getFileMetadata().getLabel()); - } catch (Exception ex) { - this.setFileName("unknown"); - } - - // This "status" is a leftover from 3.6; we don't have a use for it - // in 4.0 yet; and we may not need it at all. - // -- L.A. 4.0.2 - this.setStatus(200); - } - - // this is a Swift-specific override of the convenience method provided in the - // DataFileIO for copying a local Path (for ex., a temp file, into this DataAccess location): - - @Override - public void copyPath(Path fileSystemPath) throws IOException { - long newFileSize = -1; - - if (swiftFileObject == null || !this.canWrite()) { - open(DataAccessOption.WRITE_ACCESS); - } - - File inputFile = null; - - try { - inputFile = fileSystemPath.toFile(); - - swiftFileObject.uploadObject(inputFile); - - newFileSize = inputFile.length(); - - } catch (Exception ioex) { - String failureMsg = ioex.getMessage(); - if (failureMsg == null) { - failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject"; - } - - throw new IOException(failureMsg); - } - - // if it has uploaded successfully, we can reset the size - // of the object: - setSize(newFileSize); - - } - // @Override - // public void copyPath(Path fileSystemPath) throws IOException { - // long newFileSize = -1; - // Properties p = getSwiftProperties(); - // String swiftEndPoint = p.getProperty("swift.default.endpoint"); - // String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint); - - // if (swiftFileObject == null || !this.canWrite()) { - // open(DataAccessOption.WRITE_ACCESS); - // } - - // File inputFile = null; - - // try { - // inputFile = fileSystemPath.toFile(); - - // //@author Anuj Thakur - // swiftFileObject.uploadObject(inputFile); - // //After the files object is uploaded the identifier is changed. - // logger.info(this.swiftFileName + " " + this.swiftFolderPath); - // this.getDataFile().setStorageIdentifier( + this.swiftFolderPath + "/" + this.swiftFileName); - - // newFileSize = inputFile.length(); - - // } catch (Exception ioex) { - // String failureMsg = ioex.getMessage(); - // if (failureMsg == null) { - // failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject"; - // } - - // throw new IOException(failureMsg); - // } - - // // if it has uploaded successfully, we can reset the size - // // of the object: - // setSize(newFileSize); - - // } - - @Override - public void delete() throws IOException { - //throw new IOException("SwiftAccessIO: delete() not yet implemented in this storage driver."); - if (swiftFileObject == null) { - try { - swiftFileObject = initializeSwiftFileObject(false); - } catch (IOException ioex) { - swiftFileObject = null; - } - } - - if (swiftFileObject != null) { - swiftFileObject.delete(); - } - } - - @Override - public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException { - - throw new IOException("SwiftAccessIO: openAuxChannel() not yet implemented in this storage driver."); - } - - @Override - public boolean isAuxObjectCached(String auxItemTag) throws IOException { - throw new IOException("SwiftAccessIO: isAuxObjectCached() not yet implemented in this storage driver."); - } - - @Override - public long getAuxObjectSize(String auxItemTag) throws IOException { - throw new IOException("SwiftAccessIO: getAuxObjectSize() not yet implemented in this storage driver."); - } - - @Override - public void backupAsAux(String auxItemTag) throws IOException { - throw new IOException("SwiftAccessIO: backupAsAux() not yet implemented in this storage driver."); - } - - @Override - public String getStorageLocation() { - // What should this be, for a Swift file? - // A Swift URL? - // Or a Swift URL with an authenticated Auth token? - - return null; - } - - @Override - public Path getFileSystemPath() throws IOException { - throw new IOException("SwiftAccessIO: this is a remote AccessIO object, it has no local filesystem path associated with it."); - } - - // Auxilary helper methods, Swift-specific: - private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOException { - String storageIdentifier = this.getDataFile().getStorageIdentifier(); - - String swiftEndPoint = null; - String swiftContainer = null; - - - if (storageIdentifier.startsWith("swift://")) { - // This is a call on an already existing swift object. - - String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3); - - if (swiftStorageTokens.length != 3) { - // bad storage identifier - throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); - } - - swiftEndPoint = swiftStorageTokens[0]; - swiftContainer = swiftStorageTokens[1]; - swiftFileName = swiftStorageTokens[2]; - - if (swiftEndPoint == null || swiftContainer == null || swiftFileName == null - || "".equals(swiftEndPoint) || "".equals(swiftContainer) || "".equals(swiftFileName)) { - // all of these things need to be specified, for this to be a valid Swift location - // identifier. - throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier); - } - } else if (this.isReadAccess) { - // An attempt to call Swift driver, in a Read mode on a non-swift stored datafile - // object! - throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object."); - } else if (this.isWriteAccess) { - Properties p = getSwiftProperties(); - swiftEndPoint = p.getProperty("swift.default.endpoint"); - - //swiftFolderPath = this.getDataFile().getOwner().getDisplayName(); - String swiftFolderPathSeparator = "_"; - String authorityNoSlashes = this.getDataFile().getOwner().getAuthority().replace(this.getDataFile().getOwner().getDoiSeparator(), swiftFolderPathSeparator); - swiftFolderPath = this.getDataFile().getOwner().getProtocol() + swiftFolderPathSeparator + - authorityNoSlashes.replace(".", swiftFolderPathSeparator) + - swiftFolderPathSeparator + this.getDataFile().getOwner().getIdentifier(); - swiftFileName = storageIdentifier; - //swiftFileName = this.getDataFile().getDisplayName(); - //Storage Identifier is now updated after the object is uploaded on Swift. - this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftFolderPath+":"+swiftFileName); - } else { - throw new IOException("SwiftAccessIO: unknown access mode."); - } - // Authenticate with Swift: - - account = authenticateWithSwift(swiftEndPoint); - - /* - The containers created is swiftEndPoint concatenated with the swiftContainer - property. Creating container with certain names throws 'Unable to create - container' error on Openstack. - Any datafile with http://rdgw storage identifier i.e present on Object - store service endpoint already only needs to look-up for container using - just swiftContainer which is the concatenated name. - In future, a container for the endpoint can be created and for every - other swiftContainer Object Store pseudo-folder can be created, which is - not provide by the joss Java swift library as of yet. - */ - Container dataContainer; - - if (storageIdentifier.startsWith("swift://")) { - dataContainer = account.getContainer(swiftContainer); - } else { - dataContainer = account.getContainer(swiftFolderPath); //changed from swiftendpoint - } - - if (!dataContainer.exists()) { - if (writeAccess) { - dataContainer.create(); - //dataContainer.makePublic(); - } else { - // This is a fatal condition - it has to exist, if we were to - // read an existing object! - throw new IOException("SwiftAccessIO: container " + swiftContainer + " does not exist."); - } - } - - StoredObject fileObject = dataContainer.getObject(swiftFileName); - //file download url for public files - DataAccess.swiftFileUri = DataAccess.getSwiftFileURI(fileObject); - setRemoteUrl(DataAccess.getSwiftFileURI(fileObject)); - - logger.info(DataAccess.swiftFileUri + " success"); - - if (!writeAccess && !fileObject.exists()) { - throw new IOException("SwiftAccessIO: File object " + swiftFileName + " does not exist (Dataverse datafile id: " + this.getDataFile().getId()); - } - - return fileObject; - } - - private InputStream openSwiftFileAsInputStream() throws IOException { - InputStream in = null; - - swiftFileObject = initializeSwiftFileObject(false); - - in = swiftFileObject.downloadObjectAsInputStream(); - this.setSize(swiftFileObject.getContentLength()); - - return in; - } - - private Properties getSwiftProperties() throws IOException { - if (swiftProperties == null) { - String domainRoot = System.getProperties().getProperty("com.sun.aas.instanceRoot"); - String swiftPropertiesFile = domainRoot + File.separator + "config" + File.separator + "swift.properties"; - swiftProperties = new Properties(); - swiftProperties.load(new FileInputStream(new File(swiftPropertiesFile))); - } - - return swiftProperties; - } - - Account authenticateWithSwift(String swiftEndPoint) throws IOException { - - Properties p = getSwiftProperties(); - - // (this will throw an IOException, if the swift properties file - // is missing or corrupted) - String swiftEndPointAuthUrl = p.getProperty("swift.auth_url." + swiftEndPoint); - String swiftEndPointUsername = p.getProperty("swift.username." + swiftEndPoint); - String swiftEndPointSecretKey = p.getProperty("swift.password." + swiftEndPoint); - String swiftEndPointTenantName = p.getProperty("swift.tenant." + swiftEndPoint); - String swiftEndPointAuthMethod = p.getProperty("swift.auth_type." + swiftEndPoint); - - if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null - || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) { - // again, all of these things need to be defined, for this Swift endpoint to be - // accessible. - throw new IOException("SwiftAccessIO: no configuration available for endpoint " + swiftEndPoint); - } - - // Authenticate: - Account account = null; - - /* - This try { } now authenticates using either the KEYSTONE mechanism which uses - the tenant name in addition to the Username Password and AuthUrl OR the BASIC method - Also, the AuthUrl is now the identity service endpoint of MOC Openstack - environment instead of the Object store service endpoint. - */ - // Keystone vs. Basic - try { - if (swiftEndPointAuthMethod.equals("keystone")) { - account = new AccountFactory() - .setTenantName(swiftEndPointTenantName) - .setUsername(swiftEndPointUsername) - .setPassword(swiftEndPointSecretKey) - .setAuthUrl(swiftEndPointAuthUrl) - .createAccount(); - } else { // assume BASIC - account = new AccountFactory() - .setUsername(swiftEndPointUsername) - .setPassword(swiftEndPointSecretKey) - .setAuthUrl(swiftEndPointAuthUrl) - .setAuthenticationMethod(BASIC) - .createAccount(); - } - - } catch (Exception ex) { - ex.printStackTrace(); - throw new IOException("SwiftAccessIO: failed to authenticate " + swiftEndPointAuthMethod + " for the end point " + swiftEndPoint); - } - - return account; - } - - private boolean isWriteAccessRequested(DataAccessOption... options) throws IOException { - - for (DataAccessOption option : options) { - // In the future we may need to be able to open read-write - // Channels; no support, or use case for that as of now. - - if (option == DataAccessOption.READ_ACCESS) { - return false; - } - - if (option == DataAccessOption.WRITE_ACCESS) { - return true; - } - } - - // By default, we open the file in read mode: - return false; - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index be538442d22..f9ab137a603 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -37,7 +37,6 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.DataFileIO; import edu.harvard.iq.dataverse.dataaccess.FileAccessIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -268,14 +267,12 @@ public void addFiles (DatasetVersion version, List newFiles) { try { - logger.info("Attempting to create a new DataFileIO object for " + storageId); - DataFileIO dataAccess = DataAccess.createNewDataFileIO(dataFile, storageId); + DataFileIO dataAccess = dataFile.getAccessObject(); if (dataAccess.isLocalFile()) { localFile = true; } - - logger.info("Successfully created a new DataFileIO object."); + /* This commented-out code demonstrates how to copy bytes from a local InputStream (or a readChannel) into the @@ -314,7 +311,7 @@ from a local InputStream (or a readChannel) into the savedSuccess = true; } catch (IOException ioex) { - logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); + logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier()); } finally { if (readChannel != null) {try{readChannel.close();}catch(IOException e){}} if (writeChannel != null) {try{writeChannel.close();}catch(IOException e){}} diff --git a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java index df81d0328aa..791b376fb1e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java +++ b/src/main/java/edu/harvard/iq/dataverse/rserve/RemoteDataFrameService.java @@ -564,7 +564,7 @@ public File runDataPreprocessing(DataFile dataFile) { // send the tabular data file to the Rserve side: DataAccessRequest daReq = new DataAccessRequest(); - DataFileIO accessObject = DataAccess.getDataFileIO(dataFile, daReq); + DataFileIO accessObject = DataAccess.createDataAccessObject(dataFile, daReq); if (accessObject == null) { return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 0f87379d163..02a09edbed1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -222,11 +222,8 @@ public enum Key { Default is false; */ TwoRavensTabularView, - /** - * Whether to save data files locally in files or store them in swift - * storage, default is file. - */ - Driver, + + /** The message added to a popup upon dataset publish * From 07a469f71f0c8b3d9782813cd877b573454c2067 Mon Sep 17 00:00:00 2001 From: pjw Date: Mon, 24 Apr 2017 16:46:57 +0100 Subject: [PATCH 8/8] add entry for postgresql 9_6 --- scripts/installer/install | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/installer/install b/scripts/installer/install index 1c0289feb21..37f87634367 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -161,7 +161,8 @@ my %POSTGRES_DRIVERS = ( "9_2", "postgresql-9.1-902.jdbc4.jar", "9_3", "postgresql-9.1-902.jdbc4.jar", "9_4", "postgresql-9.1-902.jdbc4.jar", - "9_5", "postgresql-9.1-902.jdbc4.jar" + "9_5", "postgresql-9.1-902.jdbc4.jar", + "9_6", "postgresql-9.1-902.jdbc4.jar" ); # A few preliminary checks: