From 98f4f87453ddf364ad8b057b51509d02912923cd Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 3 Nov 2020 09:17:21 +0100 Subject: [PATCH 1/8] Feature/checksum maintenance (#50) This PR introduces a maintenance task that will refresh the path info db and update missing checksums --- CHANGELOG.md | 4 ++++ maintenance-tasks/checksum-maintenance/plugin.properties | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 maintenance-tasks/checksum-maintenance/plugin.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 5130ba87..ecfb7338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Currently in development + +* New maintenance task: update missing checksum one, after dss start. + ## 1.4.0 * Provide first imaging registration support with OMERO server diff --git a/maintenance-tasks/checksum-maintenance/plugin.properties b/maintenance-tasks/checksum-maintenance/plugin.properties new file mode 100644 index 00000000..beb2f66a --- /dev/null +++ b/maintenance-tasks/checksum-maintenance/plugin.properties @@ -0,0 +1,5 @@ +# Feeds pathinfo database and updates missing checksums +class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseRefreshingTask +execute-only-once = true +compute-checksum = true +time-stamp-of-youngest-data-set = 2019-12-01 00:00:00 From 0baad7adbea53ee9eb520e541281ff6238319813 Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 3 Nov 2020 10:24:11 +0100 Subject: [PATCH 2/8] Feature/checksum maintenance (#52) This PR favors the ChecksumCalculationTask over the path info db refresh task and has been confirmed to work in a openbis 18.06.2 test instance. --- maintenance-tasks/checksum-maintenance/plugin.properties | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/maintenance-tasks/checksum-maintenance/plugin.properties b/maintenance-tasks/checksum-maintenance/plugin.properties index beb2f66a..69e8ac15 100644 --- a/maintenance-tasks/checksum-maintenance/plugin.properties +++ b/maintenance-tasks/checksum-maintenance/plugin.properties @@ -1,5 +1,3 @@ -# Feeds pathinfo database and updates missing checksums -class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseRefreshingTask -execute-only-once = true -compute-checksum = true -time-stamp-of-youngest-data-set = 2019-12-01 00:00:00 +# Updates missing checksums +class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseChecksumCalculationTask +execute-only-once = true \ No newline at end of file From c8aa7a942532c1b499937a8696b3c7585660568e Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 3 Nov 2020 10:26:45 +0100 Subject: [PATCH 3/8] Update CL --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecfb7338..9878718d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## Currently in development +## 1.5.0 2020-11-03 -* New maintenance task: update missing checksum one, after dss start. +* New maintenance task: update missing checksum once, after dss starts. ## 1.4.0 From 1c67c06b1f9d80f498fd540232bd4530ab6ce97c Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 3 Nov 2020 10:29:24 +0100 Subject: [PATCH 4/8] Add new line at end of file Add new line at end of file --- maintenance-tasks/checksum-maintenance/plugin.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintenance-tasks/checksum-maintenance/plugin.properties b/maintenance-tasks/checksum-maintenance/plugin.properties index 69e8ac15..25de8e99 100644 --- a/maintenance-tasks/checksum-maintenance/plugin.properties +++ b/maintenance-tasks/checksum-maintenance/plugin.properties @@ -1,3 +1,3 @@ # Updates missing checksums class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseChecksumCalculationTask -execute-only-once = true \ No newline at end of file +execute-only-once = true From cb9a36816888e20ed99509788d76a617ff755631 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 3 Nov 2020 10:55:57 +0100 Subject: [PATCH 5/8] convert experiment id to string for v3 objects --- .../update-experiment-metadata/update.py | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/reporting-plugins/update-experiment-metadata/update.py b/reporting-plugins/update-experiment-metadata/update.py index 5c826561..d39ba2a3 100644 --- a/reporting-plugins/update-experiment-metadata/update.py +++ b/reporting-plugins/update-experiment-metadata/update.py @@ -1,5 +1,3 @@ -import smtplib -from email.mime.text import MIMEText def process(tr, parameters, tableBuilder): """Change properties of experiment @@ -9,30 +7,9 @@ def process(tr, parameters, tableBuilder): if not user == None: tr.setUserId(user) expId = parameters.get("identifier") - exp = tr.getExperimentForUpdate(expId) + exp = tr.getExperimentForUpdate(str(expId)) properties = parameters.get("properties") for prop in properties.keySet(): exp.setPropertyValue(prop, properties.get(prop)) - - #server = "smtpserv.uni-tuebingen.de" - #fromA = "notification_service@qbis.qbic.uni-tuebingen.de" - - # TODO get emails of space users - # Get it via liferay and pass it to this service ? - #toA = "mohr@informatik.uni-tuebingen.de" - #subject = "Update information for Experiment %s" % expId - #text = "Status of Experiment %s has been updated" % expId #, properties.get("Q_CURRENT_STATUS")) - - #msg = MIMEText(text) - #msg['From'] = fromA - #msg['To'] = toA - #msg['Subject'] = subject - # check for info@qbic.uni-tuebingen.de - #msg['reply-to'] = "mohr@informatik.uni-tuebingen.de" - - #smtpServer = smtplib.SMTP(server) - #smtpServer.sendmail(fromA, toA, msg.as_string()) - #smtpServer.close() - From 187bb14f2d28520ecdd70dfdcec488040b93f3f7 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 3 Nov 2020 11:15:16 +0100 Subject: [PATCH 6/8] Convert experiment id to string for v3 objects (#55) --- .../update-experiment-metadata/update.py | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/reporting-plugins/update-experiment-metadata/update.py b/reporting-plugins/update-experiment-metadata/update.py index 5c826561..d39ba2a3 100644 --- a/reporting-plugins/update-experiment-metadata/update.py +++ b/reporting-plugins/update-experiment-metadata/update.py @@ -1,5 +1,3 @@ -import smtplib -from email.mime.text import MIMEText def process(tr, parameters, tableBuilder): """Change properties of experiment @@ -9,30 +7,9 @@ def process(tr, parameters, tableBuilder): if not user == None: tr.setUserId(user) expId = parameters.get("identifier") - exp = tr.getExperimentForUpdate(expId) + exp = tr.getExperimentForUpdate(str(expId)) properties = parameters.get("properties") for prop in properties.keySet(): exp.setPropertyValue(prop, properties.get(prop)) - - #server = "smtpserv.uni-tuebingen.de" - #fromA = "notification_service@qbis.qbic.uni-tuebingen.de" - - # TODO get emails of space users - # Get it via liferay and pass it to this service ? - #toA = "mohr@informatik.uni-tuebingen.de" - #subject = "Update information for Experiment %s" % expId - #text = "Status of Experiment %s has been updated" % expId #, properties.get("Q_CURRENT_STATUS")) - - #msg = MIMEText(text) - #msg['From'] = fromA - #msg['To'] = toA - #msg['Subject'] = subject - # check for info@qbic.uni-tuebingen.de - #msg['reply-to'] = "mohr@informatik.uni-tuebingen.de" - - #smtpServer = smtplib.SMTP(server) - #smtpServer.sendmail(fromA, toA, msg.as_string()) - #smtpServer.close() - From 7f578fd05ed74e65c27219afb13e423ba00463ed Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 3 Nov 2020 11:15:53 +0100 Subject: [PATCH 7/8] Rename data folder for pooled data (#54) --- drop-boxes/register-nanopore-dropbox/register-nanopore.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drop-boxes/register-nanopore-dropbox/register-nanopore.py b/drop-boxes/register-nanopore-dropbox/register-nanopore.py index bc6b2a0c..7ae690ac 100644 --- a/drop-boxes/register-nanopore-dropbox/register-nanopore.py +++ b/drop-boxes/register-nanopore-dropbox/register-nanopore.py @@ -226,12 +226,15 @@ def registerUnclassifiedData(transaction, unclassifiedDataMap, runExperiment, cu # moves a subset of nanopore data to a new target path, needed to add fastq and fast5 subfolders to the same dataset def prepareDataFolder(incomingPath, currentPath, destinationPath, dataObject, suffix): name = dataObject.getName() + # if pooled data, folder is named using barcode and needs to be adapted + if not "_" in name: + name = name + "_" + suffix relativePath = dataObject.getRelativePath() # the source path of the currently handled data object (e.g. fast5_fail folder) sourcePath = os.path.join(os.path.dirname(currentPath), relativePath) checksumFile = createChecksumFileForFolder(incomingPath, sourcePath) # destination path containing data type (fastq or fast5), as well as the parent sample code, so pooled samples can be handled - destination = os.path.join(destinationPath, name + "_" + suffix) + destination = os.path.join(destinationPath, name) os.rename(sourcePath, destination) def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSample, openbisExperiment, currentPath, absLogPath): From d2791d4e4fe2c9e200c20a0ab2c4f223a91a2ddf Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 3 Nov 2020 11:36:02 +0100 Subject: [PATCH 8/8] adapt changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8662a8..32e81588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ## 1.5.0 2020-11-03 * New maintenance task: update missing checksum once, after dss starts. +* Fix for nanopore registration: rename folders for pooling case +* Fix for experiment update: force identifier into a string to support v3 API objects ## 1.4.1 2020-11-03