diff --git a/lib/ontologies_linked_data/models/ontology_submission.rb b/lib/ontologies_linked_data/models/ontology_submission.rb index c60b6901..a79abb09 100644 --- a/lib/ontologies_linked_data/models/ontology_submission.rb +++ b/lib/ontologies_linked_data/models/ontology_submission.rb @@ -15,7 +15,6 @@ class OntologySubmission < LinkedData::Models::Base include LinkedData::Concerns::SubmissionProcessable include LinkedData::Concerns::OntologySubmission::MetadataExtractor - FILES_TO_DELETE = ['labels.ttl', 'mappings.ttl', 'obsolete.ttl', 'owlapi.xrdf', 'errors.log'] FLAT_ROOTS_LIMIT = 1000 model :ontology_submission, name_with: lambda { |s| submission_id_generator(s) } diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb index 1e69df0f..f5e18e34 100644 --- a/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb @@ -2,29 +2,49 @@ module LinkedData module Services class OntologySubmissionArchiver < OntologySubmissionProcess - FILES_TO_DELETE = %w[labels.ttl mappings.ttl obsolete.ttl owlapi.xrdf errors.log] - + FILES_TO_DELETE = ['labels.ttl', 'mappings.ttl', 'obsolete.ttl', 'owlapi.xrdf', 'errors.log'] + FOLDERS_TO_DELETE = ['unzipped'] + FILE_SIZE_ZIPPING_THRESHOLD = 100 * 1024 * 1024 # 100MB def process - submission_archive + archive_submission end private - def submission_archive + + def archive_submission + @submission.ontology.bring(:submissions) + submissions = @submission.ontology.submissions + return if submissions.nil? + + submissions.each { |s| s.bring(:submissionId) } + submission = submissions.sort { |a, b| b.submissionId <=> a.submissionId }.first + + return unless @submission.submissionId < submission.submissionId + @submission.submissionStatus = nil status = LinkedData::Models::SubmissionStatus.find("ARCHIVED").first @submission.add_submission_status(status) + @submission.unindex # Delete everything except for original ontology file. - @submission.ontology.bring(:submissions) - submissions = @submission.ontology.submissions - unless submissions.nil? - submissions.each { |s| s.bring(:submissionId) } - submission = submissions.sort { |a, b| b.submissionId <=> a.submissionId }.first - # Don't perform deletion if this is the most recent submission. - delete_old_submission_files if @submission.submissionId < submission.submissionId - end + delete_old_submission_files + @submission.uploadFilePath = zip_submission_uploaded_file + end + + def zip_submission_uploaded_file + @submission.bring(:uploadFilePath) if @submission.bring?(:uploadFilePath) + return @submission.uploadFilePath if @submission.zipped? + + return @submission.uploadFilePath if @submission.uploadFilePath.nil? || @submission.uploadFilePath.empty? + + return @submission.uploadFilePath if File.size(@submission.uploadFilePath) < FILE_SIZE_ZIPPING_THRESHOLD + + old_path = @submission.uploadFilePath + zip_file = Utils::FileHelpers.zip_file(old_path) + FileUtils.rm(old_path, force: true) + zip_file end def delete_old_submission_files @@ -33,10 +53,11 @@ def delete_old_submission_files submission_files.push(@submission.csv_path) submission_files.push(@submission.parsing_log_path) unless @submission.parsing_log_path.nil? FileUtils.rm(submission_files, force: true) + submission_folders = FOLDERS_TO_DELETE.map { |f| File.join(path_to_repo, f) } + submission_folders.each { |d| FileUtils.remove_dir(d) if File.directory?(d) } end end - end end diff --git a/lib/ontologies_linked_data/services/submission_process/submission_process.rb b/lib/ontologies_linked_data/services/submission_process/submission_process.rb index 4e7433e8..3c44acd4 100644 --- a/lib/ontologies_linked_data/services/submission_process/submission_process.rb +++ b/lib/ontologies_linked_data/services/submission_process/submission_process.rb @@ -1,4 +1,4 @@ -module LinkedData + module LinkedData module Services class OntologySubmissionProcess @@ -7,6 +7,10 @@ def initialize(submission) end def process(logger, options = {}) + call + end + + def call raise NotImplementedError end end diff --git a/lib/ontologies_linked_data/utils/file.rb b/lib/ontologies_linked_data/utils/file.rb index f740300d..c7a25d5c 100644 --- a/lib/ontologies_linked_data/utils/file.rb +++ b/lib/ontologies_linked_data/utils/file.rb @@ -7,6 +7,14 @@ module LinkedData module Utils module FileHelpers + + class GzipFile + attr_accessor :name + def initialize(gz) + self.name = gz.orig_name + end + end + def self.zip?(file_path) file_path = file_path.to_s @@ -78,6 +86,21 @@ def self.unzip(file_path, dst_folder) extracted_files end + def self.zip_file(file_path) + return file_path if self.zip?(file_path) + + zip_file_path = "#{file_path}.zip" + Zip::File.open(zip_file_path, Zip::File::CREATE) do |zipfile| + # Add the file to the zip + begin + zipfile.add(File.basename(file_path), file_path) + rescue Zip::EntryExistsError + end + + end + zip_file_path + end + def self.automaster?(path, format) self.automaster(path, format) != nil end