DB/Web/Importer: Store Scan Files in File Object Storage when configured

Add new Config entries for:
Store Scan Files to File Object Storage Service
Allow Download of Scan Files from File Object Service.

DB changes
Importer Changes
Webapp Changes

In Project page Scan files section add [Download]
In Search details, click on scan file downloads it

database_scripts/install/001_create_empty_database.sql

@@ -481,20 +481,64 @@ CREATE TABLE  user_access_level_label_description_tbl (
 ENGINE = InnoDB;
 
 
+-- -----------------------------------------------------
+-- Table file_object_storage_main_entry_file_type_lookup_tbl
+-- -----------------------------------------------------
+CREATE TABLE  file_object_storage_main_entry_file_type_lookup_tbl (
+  id INT UNSIGNED NOT NULL,
+  description VARCHAR(300) NOT NULL,
+  create_date DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (id))
+ENGINE = InnoDB;
+
+CREATE UNIQUE INDEX file_object_storage_api_key_UNIQUE ON file_object_storage_main_entry_file_type_lookup_tbl (description ASC) VISIBLE;
+
+
+-- -----------------------------------------------------
+-- Table file_object_storage_main_entry_tbl
+-- -----------------------------------------------------
+CREATE TABLE  file_object_storage_main_entry_tbl (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  file_type_id INT UNSIGNED NOT NULL,
+  file_object_storage_api_key VARCHAR(300) NOT NULL,
+  create_date DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (id),
+  CONSTRAINT fk_file_object_storage_main_entry_tbl_1
+    FOREIGN KEY (file_type_id)
+    REFERENCES file_object_storage_main_entry_file_type_lookup_tbl (id)
+    ON DELETE NO ACTION
+    ON UPDATE NO ACTION)
+ENGINE = InnoDB
+DEFAULT CHARACTER SET = latin1
+COLLATE = latin1_bin;
+
+CREATE UNIQUE INDEX file_object_storage_api_key_UNIQUE ON file_object_storage_main_entry_tbl (file_object_storage_api_key ASC) VISIBLE;
+
+CREATE INDEX fk_file_object_storage_main_entry_tbl_1_idx ON file_object_storage_main_entry_tbl (file_type_id ASC) VISIBLE;
+
+
 -- -----------------------------------------------------
 -- Table scan_file_tbl
 -- -----------------------------------------------------
 CREATE TABLE  scan_file_tbl (
   id INT UNSIGNED NOT NULL AUTO_INCREMENT,
   spectral_storage_api_key VARCHAR(300) NOT NULL,
+  file_object_storage_main_entry_id_fk INT UNSIGNED NULL,
   create_date DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  PRIMARY KEY (id))
+  PRIMARY KEY (id),
+  CONSTRAINT scan_file_tbl_fos_meid
+    FOREIGN KEY (file_object_storage_main_entry_id_fk)
+    REFERENCES file_object_storage_main_entry_tbl (id)
+    ON DELETE NO ACTION
+    ON UPDATE NO ACTION)
 ENGINE = InnoDB
 DEFAULT CHARACTER SET = latin1
 COLLATE = latin1_bin;
 
 CREATE UNIQUE INDEX spectral_storage_api_key_UNIQUE ON scan_file_tbl (spectral_storage_api_key ASC) VISIBLE;
 
+CREATE INDEX scan_file_tbl_fos_meid_idx ON scan_file_tbl (file_object_storage_main_entry_id_fk ASC) VISIBLE;
+
 
 -- -----------------------------------------------------
 -- Table search_programs_per_search_tbl
@@ -3523,42 +3567,6 @@ ENGINE = InnoDB;
 CREATE INDEX fk_import_and_pipeline_run_tracking_status_history_tbl_1_idx ON import_and_pipeline_run_tracking_status_history_tbl (import_and_pipeline_run_tracking_id ASC) VISIBLE;
 
 
--- -----------------------------------------------------
--- Table file_object_storage_main_entry_file_type_lookup_tbl
--- -----------------------------------------------------
-CREATE TABLE  file_object_storage_main_entry_file_type_lookup_tbl (
-  id INT UNSIGNED NOT NULL,
-  description VARCHAR(300) NOT NULL,
-  create_date DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  PRIMARY KEY (id))
-ENGINE = InnoDB;
-
-CREATE UNIQUE INDEX file_object_storage_api_key_UNIQUE ON file_object_storage_main_entry_file_type_lookup_tbl (description ASC) VISIBLE;
-
-
--- -----------------------------------------------------
--- Table file_object_storage_main_entry_tbl
--- -----------------------------------------------------
-CREATE TABLE  file_object_storage_main_entry_tbl (
-  id INT UNSIGNED NOT NULL AUTO_INCREMENT,
-  file_type_id INT UNSIGNED NOT NULL,
-  file_object_storage_api_key VARCHAR(300) NOT NULL,
-  create_date DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  PRIMARY KEY (id),
-  CONSTRAINT fk_file_object_storage_main_entry_tbl_1
-    FOREIGN KEY (file_type_id)
-    REFERENCES file_object_storage_main_entry_file_type_lookup_tbl (id)
-    ON DELETE NO ACTION
-    ON UPDATE NO ACTION)
-ENGINE = InnoDB
-DEFAULT CHARACTER SET = latin1
-COLLATE = latin1_bin;
-
-CREATE UNIQUE INDEX file_object_storage_api_key_UNIQUE ON file_object_storage_main_entry_tbl (file_object_storage_api_key ASC) VISIBLE;
-
-CREATE INDEX fk_file_object_storage_main_entry_tbl_1_idx ON file_object_storage_main_entry_tbl (file_type_id ASC) VISIBLE;
-
-
 -- -----------------------------------------------------
 -- Table file_object_storage_to_search_tbl
 -- -----------------------------------------------------

database_scripts/install/090_docker_compose.sql

@@ -9,6 +9,8 @@ INSERT INTO config_system_tbl (config_key, config_value) VALUES ('email_smtp_ser
 INSERT INTO config_system_tbl (config_key, config_value) VALUES ('footer_center_of_page_html', 'Limelight Docker created by Michael Riffle (<a href="mailto:mriffle@uw.edu" target="_top">mriffle@uw.edu</a>)');
 INSERT INTO config_system_tbl (config_key, config_value) VALUES ('file_import_limelight_xml_scans_temp_dir', '/data/limelight_upload');
 INSERT INTO config_system_tbl (config_key, config_value) VALUES ('scan_file_import_allowed_via_web_submit', 'true');
+INSERT INTO config_system_tbl (config_key, config_value) VALUES ('scan_file_save_to_file_object_storage', 'true');
+INSERT INTO config_system_tbl (config_key, config_value) VALUES ('scan_file_download_from_file_object_storage_allowed', 'true');
 INSERT INTO config_system_tbl (config_key, config_value) VALUES ('import_delete_uploaded_files_after_import', 'true');
 INSERT INTO config_system_tbl (config_key, config_value) VALUES ('run_import_extra_emails_to_send_to', 'root@localhost');
 INSERT INTO config_system_tbl (config_key, config_value) VALUES ('admin_email_address', 'root@localhost');

database_scripts/version_upgrades/6/version_upgrade.sql

@@ -322,6 +322,21 @@ COMMENT = 'When Copy a search to new project, track the from and to project sear
 CREATE INDEX project_search_id_new ON project_search_copy_from_to_ids_tbl (project_search_id_new ASC) VISIBLE;
 
 
+-- -----------------------------------------------------
+-- ALTER TABLE scan_file_tbl
+-- -----------------------------------------------------
+ALTER TABLE scan_file_tbl 
+ADD COLUMN file_object_storage_main_entry_id_fk INT UNSIGNED NULL AFTER spectral_storage_api_key,
+ADD INDEX scan_file_tbl_fos_meid_idx (file_object_storage_main_entry_id_fk ASC) VISIBLE;
+
+ALTER TABLE scan_file_tbl 
+ADD CONSTRAINT scan_file_tbl_fos_meid
+  FOREIGN KEY (file_object_storage_main_entry_id_fk)
+  REFERENCES file_object_storage_main_entry_tbl (id)
+  ON DELETE NO ACTION
+  ON UPDATE NO ACTION;
+
+
 
 --  !!!!!!!   UPGRADE SQL END    !!!!!!!!
 

limelight_importer/src/main/java/org/yeastrc/limelight/limelight_importer/dao/ScanFileDAO_Importer.java

@@ -89,6 +89,67 @@ public Integer getScanFileIdForSpectralStorageAPIKey( String sequence, Connectio
 		return id;
 	}
 
+	/**
+	 * Get the id for the supplied spectral_storage_api_key from the database. 
+	 * @param id
+	 * @return null if not found
+	 * @throws Exception
+	 */
+	public Integer get_FileObjectStorage_MainEntry_Id_For_ScanFileId( int id ) throws Exception {
+
+		Integer result = null;
+
+		final String sql = "SELECT file_object_storage_main_entry_id_fk FROM scan_file_tbl WHERE id = ? ORDER BY id LIMIT 1";
+
+		try ( Connection dbConnection = ImportRunImporterDBConnectionFactory.getMainSingletonInstance().getConnection() ) {
+			try ( PreparedStatement pstmt = dbConnection.prepareStatement( sql ) ) {
+				pstmt.setInt( 1, id );
+
+				try ( ResultSet rs = pstmt.executeQuery() ) {
+					if( rs.next() ) {
+						int fieldValue = rs.getInt( "file_object_storage_main_entry_id_fk" );
+						if ( ! rs.wasNull() ) {
+							result = fieldValue;
+						}
+					}
+				}
+			}
+		} catch ( Exception e ) {
+			log.error( "ERROR: get_FileObjectStorage_MainEntry_Id_For_ScanFileId(...) sql: " + sql, e );
+			throw e;
+		}
+
+		return result;
+	}
+
+
+	/**
+	 * Update record set fileObjectStorage_MainEntry_Id where id
+	 * 
+	 * @param fileObjectStorage_MainEntry_Id
+	 * @param id
+	 * @return null if not found
+	 * @throws Exception
+	 */
+	public void update_FileObjectStorage_MainEntry_Id_For_ScanFileId( int fileObjectStorage_MainEntry_Id, int id ) throws Exception {
+
+		final String sql = "UPDATE scan_file_tbl SET file_object_storage_main_entry_id_fk = ? WHERE id = ?";
+
+		try ( Connection dbConnection = ImportRunImporterDBConnectionFactory.getMainSingletonInstance().getConnection() ) {
+			try ( PreparedStatement pstmt = dbConnection.prepareStatement( sql ) ) {
+
+				pstmt.setInt( 1, fileObjectStorage_MainEntry_Id );
+
+				pstmt.setInt( 2, id );
+
+				pstmt.executeUpdate();
+			}
+		} catch ( Exception e ) {
+			log.error( "ERROR: get_FileObjectStorage_MainEntry_Id_For_ScanFileId(...). id: " + id + ", fileObjectStorage_MainEntry_Id: " + fileObjectStorage_MainEntry_Id + ",  sql: " + sql, e );
+			throw e;
+		}
+
+	}
 
 	/**
 	 * @param item
@@ -107,9 +168,9 @@ public void saveToDatabase( ScanFileDTO item, SkipLogInsertException skipLogInse
 	private final static String INSERT_SQL = 
 
 			"INSERT INTO scan_file_tbl "
-			+ "( spectral_storage_api_key ) "
+			+ "( spectral_storage_api_key, file_object_storage_main_entry_id_fk ) "
 
-			+ "VALUES (?)";
+			+ "VALUES (?, ?)";
 
 	/**
 	 * @param item
@@ -120,9 +181,21 @@ public void saveToDatabase( ScanFileDTO item, SkipLogInsertException skipLogInse
 		final String sql = INSERT_SQL;
 
 		try ( PreparedStatement pstmt = dbConnection.prepareStatement( sql, Statement.RETURN_GENERATED_KEYS ) ) {
+
 			int counter = 0;
+
 			counter++;
 			pstmt.setString( counter, item.getSpectralStorageAPIKey() );
+
+			counter++;
+
+			if ( item.getFileObjectStorage_MainEntry_Id() != null ) {
+				pstmt.setInt( counter, item.getFileObjectStorage_MainEntry_Id() );
+			} else {
+				pstmt.setNull( counter, java.sql.Types.INTEGER );
+			}
+
+
 			pstmt.executeUpdate();
 
 			try ( ResultSet rs = pstmt.getGeneratedKeys() ) {

limelight_importer/src/main/java/org/yeastrc/limelight/limelight_importer/importer_core_entry_point/ImporterCoreEntryPoint.java

@@ -132,7 +132,7 @@ public int doImport(
 			ScanFileFileContainer_AllEntries scanFileFileContainer_AllEntries,
 			Boolean skipPopulatingPathOnSearchLineOptChosen, SearchStatistics_General_SavedToDB searchStatistics_General_SavedToDB
 			) throws Exception, LimelightImporterProjectNotAllowImportException, LimelightImporterLimelightXMLDeserializeFailException {
-		
+
 		LimelightInput limelightInputForImport = null;
 		if ( limelightInputForImportParam != null ) {
 			limelightInputForImport = limelightInputForImportParam;

limelight_importer/src/main/java/org/yeastrc/limelight/limelight_importer/objects/FileObjectStorage_FileContainer.java

@@ -37,6 +37,10 @@ public class FileObjectStorage_FileContainer {
 	 * Populated when running the Import from the Run Importer Process
 	 */
 	private FileImportTrackingSingleFileDTO fileImportTrackingSingleFileDTO;
+
+
+	private Integer id_InDBTable_file_object_storage_main_entry_tbl;
+
 
 	/**
 	 * constructor
@@ -75,4 +79,13 @@ public void setFileImportTrackingSingleFileDTO(FileImportTrackingSingleFileDTO f
 		this.fileImportTrackingSingleFileDTO = fileImportTrackingSingleFileDTO;
 	}
 
+	public Integer getId_InDBTable_file_object_storage_main_entry_tbl() {
+		return id_InDBTable_file_object_storage_main_entry_tbl;
+	}
+
+	public void setId_InDBTable_file_object_storage_main_entry_tbl(
+			Integer id_InDBTable_file_object_storage_main_entry_tbl) {
+		this.id_InDBTable_file_object_storage_main_entry_tbl = id_InDBTable_file_object_storage_main_entry_tbl;
+	}
+
 }

limelight_importer/src/main/java/org/yeastrc/limelight/limelight_importer/objects/ScanFileFileContainer.java

@@ -44,6 +44,11 @@ public class ScanFileFileContainer {
 	 * Populated when running the Import from the Run Importer Process
 	 */
 	private FileImportTrackingSingleFileDTO scanFileDBRecord;
+
+	/**
+	 * Populate when stored in File Object Storage
+	 */
+	private FileObjectStorage_FileContainer fileObjectStorage_FileContainer;
 
 	/**
 	 * constructor
@@ -147,4 +152,14 @@ public void setScanFilename_NoSuffix(String scanFilename_NoSuffix) {
 	}
 
 
+	public FileObjectStorage_FileContainer getFileObjectStorage_FileContainer() {
+		return fileObjectStorage_FileContainer;
+	}
+
+
+	public void setFileObjectStorage_FileContainer(FileObjectStorage_FileContainer fileObjectStorage_FileContainer) {
+		this.fileObjectStorage_FileContainer = fileObjectStorage_FileContainer;
+	}
+
+
 }

limelight_importer/src/main/java/org/yeastrc/limelight/limelight_importer/objects/SearchScanFileEntry.java

@@ -40,6 +40,8 @@ public class SearchScanFileEntry {
 	private SearchScanFileImporterDTO searchScanFileImporterDTO;
 	private ScanFileDTO scanFileDTO;
 
+	private ScanFileFileContainer scanFileFileContainer;
+
 	/**
 	 * Package Private constructor
 	 */
@@ -93,5 +95,13 @@ public List<Integer> getScanNumbersFromPSMs() {
 	public void setScanNumbersFromPSMs(List<Integer> scanNumbersFromPSMs) {
 		this.scanNumbersFromPSMs = scanNumbersFromPSMs;
 	}
+
+	public ScanFileFileContainer getScanFileFileContainer() {
+		return scanFileFileContainer;
+	}
+
+	public void setScanFileFileContainer(ScanFileFileContainer scanFileFileContainer) {
+		this.scanFileFileContainer = scanFileFileContainer;
+	}
 
 }