The new framework for deleting the physical file outside of the

database delete transaction. (#5535)

src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java

@@ -1,14 +1,17 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
+import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.FileSortFieldAndOrder;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import java.io.IOException;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -1578,4 +1581,14 @@ public boolean isGlobalIdUnique(String userIdentifier, DataFile datafile, Global
         return u;
     }
 
+    public void finalizeFileDelete(Long dataFileId, String storageLocation) throws IOException {
+        // Verify that the DataFile no longer exists: 
+        if (find(dataFileId) != null) {
+            throw new IOException("Attempted to permanently delete a physical file still associated with an existing DvObject "
+                    + "(id: " + dataFileId + ", location: " + storageLocation);
+        }
+        StorageIO directStorageAccess = DataAccess.getDirectStorageIO(storageLocation);
+        directStorageAccess.delete();
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -2521,6 +2521,9 @@ public void deleteFiles() {
                         if (fmd.getDataFile().equals(dataset.getThumbnailFile())) {
                             dataset.setThumbnailFile(null);
                         }
+                        /* It should not be possible to get here if this file 
+                           is not in fact released! - so the code block below 
+                           is not needed.
                         //if not published then delete identifier
                         if (!fmd.getDataFile().isReleased()){
                             try{
@@ -2529,7 +2532,7 @@ public void deleteFiles() {
                                  //this command is here to delete the identifier of unreleased files
                                  //if it fails then a reserved identifier may still be present on the remote provider
                             }                           
-                        }
+                        } */
                         fmit.remove();
                         break;
                     }

src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java

@@ -9,6 +9,7 @@
 import edu.harvard.iq.dataverse.datasetutility.FileReplaceException;
 import edu.harvard.iq.dataverse.datasetutility.FileReplacePageHelper;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
+import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil;
 import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse;
 import edu.harvard.iq.dataverse.dataset.DatasetThumbnail;
@@ -1291,20 +1292,51 @@ public String save() {
 
                 if (!fmd.getDataFile().isReleased()) {
                     // if file is draft (ie. new to this version, delete; otherwise just remove filemetadata object)
-                    try {
-                        commandEngine.submit(new DeleteDataFileCommand(fmd.getDataFile(), dvRequestService.getDataverseRequest()));
-                        dataset.getFiles().remove(fmd.getDataFile());
-                        workingVersion.getFileMetadatas().remove(fmd);
-                        // added this check to handle issue where you could not deleter a file that shared a category with a new file
-                        // the relationship does not seem to cascade, yet somehow it was trying to merge the filemetadata
-                        // todo: clean this up some when we clean the create / update dataset methods
-                        for (DataFileCategory cat : dataset.getCategories()) {
-                            cat.getFileMetadatas().remove(fmd);
+                    boolean deleteCommandSuccess = false;
+                    Long dataFileId = fmd.getDataFile().getId();
+                    String storageLocation = null;
+
+                    if (dataFileId != null) { // is this necessary?
+                        try {
+                            StorageIO<DataFile> storageIO = fmd.getDataFile().getStorageIO();
+                            storageIO.open();
+                            storageLocation = storageIO.getStorageLocation();
+                        } catch (IOException ioex) {
+                            // something potentially wrong with the physical file
+                            // or connection to the physical storage? 
+                            // we'll still try to delete the datafile from the database
+                        }
+                        try {
+                            commandEngine.submit(new DeleteDataFileCommand(fmd.getDataFile(), dvRequestService.getDataverseRequest()));
+                            dataset.getFiles().remove(fmd.getDataFile());
+                            workingVersion.getFileMetadatas().remove(fmd);
+                            // added this check to handle an issue where you could not delete a file that shared a category with a new file
+                            // the relationship does not seem to cascade, yet somehow it was trying to merge the filemetadata
+                            // todo: clean this up some when we clean the create / update dataset methods
+                            for (DataFileCategory cat : dataset.getCategories()) {
+                                cat.getFileMetadatas().remove(fmd);
+                            }
+                            deleteCommandSuccess = true;
+                        } catch (CommandException cmde) {
+                            // TODO: 
+                            // add diagnostics reporting for individual data files that 
+                            // we failed to delete.
+                            logger.info("Failed to delete DataFile id=" + dataFileId + " from the database; " + cmde.getMessage());
+                        }
+                        if (deleteCommandSuccess) {
+                            if (storageLocation != null) {
+                                // Finalize the delete of the physical file 
+                                // (File service will double-check that the datafile no 
+                                // longer exists in the database, before proceeding to 
+                                // delete the physical file)
+                                try {
+                                    datafileService.finalizeFileDelete(dataFileId, storageLocation);
+                                } catch (IOException ioex) {
+                                    logger.info("Failed to delete the physical file associated with the deleted datafile id="
+                                            + dataFileId + ", storage location: " + storageLocation);
+                                }
+                            }
                         }
-                    } catch (CommandException cmde) {
-                        // TODO: 
-                        // add diagnostics reporting for individual data files that 
-                        // we failed to delete.
                     }
                 } else {
                     datafileService.removeFileMetadata(fmd);