From 461b81e90e1b8fd4f4f9bed3627aefd58e0a9d48 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 2 Nov 2020 17:21:47 +0100 Subject: [PATCH 01/11] make imgag dropbox raise exception if unknown data types arrive --- drop-boxes/register-imgag-dropbox/register-imgag.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drop-boxes/register-imgag-dropbox/register-imgag.py b/drop-boxes/register-imgag-dropbox/register-imgag.py index a97ee8e8..02896d4f 100644 --- a/drop-boxes/register-imgag-dropbox/register-imgag.py +++ b/drop-boxes/register-imgag-dropbox/register-imgag.py @@ -596,13 +596,14 @@ def process(transaction): print rawFile if rawFile.endswith("vcf") or rawFile.endswith("vcf.gz"): vcfs.append(rawFile) - if rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"): + elif rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"): fastqs.append(rawFile) - if rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"): + elif rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"): gsvars.append(rawFile) - if rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"): + elif rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"): tsvs.append(rawFile) - + else: + raise Exception(rawFile + " is of an unsupported format") #if rawFiles[0].endswith("vcf") or rawFiles[0].endswith("vcf.gz"): # datasetSample = find_and_register_vcf(transaction, jsonContent) From 1b51c234f06356d64fa3aaca8491ff663a358441 Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 3 Nov 2020 09:18:38 +0100 Subject: [PATCH 02/11] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5130ba87..5190ed1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.4.1 2020-11-03 + +* Imgag dropbox: raise an exception, if files of unknown type are part of the transaction + ## 1.4.0 * Provide first imaging registration support with OMERO server From 468887b121b25966d8de6541bf20383baf7f2ddc Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 3 Nov 2020 10:24:57 +0100 Subject: [PATCH 03/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 83afc705..a34ef4d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/omero-portlet) +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/etl-scripts) ![Python Language](https://img.shields.io/badge/language-python-blue.svg) ![License](https://img.shields.io/github/license/qbicsoftware/etl-scripts) [![DOI](https://zenodo.org/badge/45912621.svg)](https://zenodo.org/badge/latestdoi/45912621) From c8aa7a942532c1b499937a8696b3c7585660568e Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 3 Nov 2020 10:26:45 +0100 Subject: [PATCH 04/11] Update CL --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecfb7338..9878718d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## Currently in development +## 1.5.0 2020-11-03 -* New maintenance task: update missing checksum one, after dss start. +* New maintenance task: update missing checksum once, after dss starts. ## 1.4.0 From 1c67c06b1f9d80f498fd540232bd4530ab6ce97c Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 3 Nov 2020 10:29:24 +0100 Subject: [PATCH 05/11] Add new line at end of file Add new line at end of file --- maintenance-tasks/checksum-maintenance/plugin.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintenance-tasks/checksum-maintenance/plugin.properties b/maintenance-tasks/checksum-maintenance/plugin.properties index 69e8ac15..25de8e99 100644 --- a/maintenance-tasks/checksum-maintenance/plugin.properties +++ b/maintenance-tasks/checksum-maintenance/plugin.properties @@ -1,3 +1,3 @@ # Updates missing checksums class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseChecksumCalculationTask -execute-only-once = true \ No newline at end of file +execute-only-once = true From 187bb14f2d28520ecdd70dfdcec488040b93f3f7 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 3 Nov 2020 11:15:16 +0100 Subject: [PATCH 06/11] Convert experiment id to string for v3 objects (#55) --- .../update-experiment-metadata/update.py | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/reporting-plugins/update-experiment-metadata/update.py b/reporting-plugins/update-experiment-metadata/update.py index 5c826561..d39ba2a3 100644 --- a/reporting-plugins/update-experiment-metadata/update.py +++ b/reporting-plugins/update-experiment-metadata/update.py @@ -1,5 +1,3 @@ -import smtplib -from email.mime.text import MIMEText def process(tr, parameters, tableBuilder): """Change properties of experiment @@ -9,30 +7,9 @@ def process(tr, parameters, tableBuilder): if not user == None: tr.setUserId(user) expId = parameters.get("identifier") - exp = tr.getExperimentForUpdate(expId) + exp = tr.getExperimentForUpdate(str(expId)) properties = parameters.get("properties") for prop in properties.keySet(): exp.setPropertyValue(prop, properties.get(prop)) - - #server = "smtpserv.uni-tuebingen.de" - #fromA = "notification_service@qbis.qbic.uni-tuebingen.de" - - # TODO get emails of space users - # Get it via liferay and pass it to this service ? - #toA = "mohr@informatik.uni-tuebingen.de" - #subject = "Update information for Experiment %s" % expId - #text = "Status of Experiment %s has been updated" % expId #, properties.get("Q_CURRENT_STATUS")) - - #msg = MIMEText(text) - #msg['From'] = fromA - #msg['To'] = toA - #msg['Subject'] = subject - # check for info@qbic.uni-tuebingen.de - #msg['reply-to'] = "mohr@informatik.uni-tuebingen.de" - - #smtpServer = smtplib.SMTP(server) - #smtpServer.sendmail(fromA, toA, msg.as_string()) - #smtpServer.close() - From 7f578fd05ed74e65c27219afb13e423ba00463ed Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 3 Nov 2020 11:15:53 +0100 Subject: [PATCH 07/11] Rename data folder for pooled data (#54) --- drop-boxes/register-nanopore-dropbox/register-nanopore.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drop-boxes/register-nanopore-dropbox/register-nanopore.py b/drop-boxes/register-nanopore-dropbox/register-nanopore.py index bc6b2a0c..7ae690ac 100644 --- a/drop-boxes/register-nanopore-dropbox/register-nanopore.py +++ b/drop-boxes/register-nanopore-dropbox/register-nanopore.py @@ -226,12 +226,15 @@ def registerUnclassifiedData(transaction, unclassifiedDataMap, runExperiment, cu # moves a subset of nanopore data to a new target path, needed to add fastq and fast5 subfolders to the same dataset def prepareDataFolder(incomingPath, currentPath, destinationPath, dataObject, suffix): name = dataObject.getName() + # if pooled data, folder is named using barcode and needs to be adapted + if not "_" in name: + name = name + "_" + suffix relativePath = dataObject.getRelativePath() # the source path of the currently handled data object (e.g. fast5_fail folder) sourcePath = os.path.join(os.path.dirname(currentPath), relativePath) checksumFile = createChecksumFileForFolder(incomingPath, sourcePath) # destination path containing data type (fastq or fast5), as well as the parent sample code, so pooled samples can be handled - destination = os.path.join(destinationPath, name + "_" + suffix) + destination = os.path.join(destinationPath, name) os.rename(sourcePath, destination) def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSample, openbisExperiment, currentPath, absLogPath): From ab8997edf3838d45de1acac0b488d51958e20704 Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 3 Nov 2020 12:01:27 +0100 Subject: [PATCH 08/11] Release/1.5.0 (#53) * Feature/checksum maintenance (#50) This PR introduces a maintenance task that will refresh the path info db and update missing checksums * Feature/checksum maintenance (#52) This PR favors the ChecksumCalculationTask over the path info db refresh task and has been confirmed to work in a openbis 18.06.2 test instance. * Update CL * Add new line at end of file Add new line at end of file * convert experiment id to string for v3 objects * Convert experiment id to string for v3 objects (#55) * Rename data folder for pooled data (#54) * adapt changelog Co-authored-by: wow-such-code --- CHANGELOG.md | 7 ++++++ .../register-nanopore.py | 5 +++- .../checksum-maintenance/plugin.properties | 3 +++ .../update-experiment-metadata/update.py | 25 +------------------ 4 files changed, 15 insertions(+), 25 deletions(-) create mode 100644 maintenance-tasks/checksum-maintenance/plugin.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 5190ed1a..32e81588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog + +## 1.5.0 2020-11-03 + +* New maintenance task: update missing checksum once, after dss starts. +* Fix for nanopore registration: rename folders for pooling case +* Fix for experiment update: force identifier into a string to support v3 API objects + ## 1.4.1 2020-11-03 * Imgag dropbox: raise an exception, if files of unknown type are part of the transaction diff --git a/drop-boxes/register-nanopore-dropbox/register-nanopore.py b/drop-boxes/register-nanopore-dropbox/register-nanopore.py index bc6b2a0c..7ae690ac 100644 --- a/drop-boxes/register-nanopore-dropbox/register-nanopore.py +++ b/drop-boxes/register-nanopore-dropbox/register-nanopore.py @@ -226,12 +226,15 @@ def registerUnclassifiedData(transaction, unclassifiedDataMap, runExperiment, cu # moves a subset of nanopore data to a new target path, needed to add fastq and fast5 subfolders to the same dataset def prepareDataFolder(incomingPath, currentPath, destinationPath, dataObject, suffix): name = dataObject.getName() + # if pooled data, folder is named using barcode and needs to be adapted + if not "_" in name: + name = name + "_" + suffix relativePath = dataObject.getRelativePath() # the source path of the currently handled data object (e.g. fast5_fail folder) sourcePath = os.path.join(os.path.dirname(currentPath), relativePath) checksumFile = createChecksumFileForFolder(incomingPath, sourcePath) # destination path containing data type (fastq or fast5), as well as the parent sample code, so pooled samples can be handled - destination = os.path.join(destinationPath, name + "_" + suffix) + destination = os.path.join(destinationPath, name) os.rename(sourcePath, destination) def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSample, openbisExperiment, currentPath, absLogPath): diff --git a/maintenance-tasks/checksum-maintenance/plugin.properties b/maintenance-tasks/checksum-maintenance/plugin.properties new file mode 100644 index 00000000..25de8e99 --- /dev/null +++ b/maintenance-tasks/checksum-maintenance/plugin.properties @@ -0,0 +1,3 @@ +# Updates missing checksums +class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseChecksumCalculationTask +execute-only-once = true diff --git a/reporting-plugins/update-experiment-metadata/update.py b/reporting-plugins/update-experiment-metadata/update.py index 5c826561..d39ba2a3 100644 --- a/reporting-plugins/update-experiment-metadata/update.py +++ b/reporting-plugins/update-experiment-metadata/update.py @@ -1,5 +1,3 @@ -import smtplib -from email.mime.text import MIMEText def process(tr, parameters, tableBuilder): """Change properties of experiment @@ -9,30 +7,9 @@ def process(tr, parameters, tableBuilder): if not user == None: tr.setUserId(user) expId = parameters.get("identifier") - exp = tr.getExperimentForUpdate(expId) + exp = tr.getExperimentForUpdate(str(expId)) properties = parameters.get("properties") for prop in properties.keySet(): exp.setPropertyValue(prop, properties.get(prop)) - - #server = "smtpserv.uni-tuebingen.de" - #fromA = "notification_service@qbis.qbic.uni-tuebingen.de" - - # TODO get emails of space users - # Get it via liferay and pass it to this service ? - #toA = "mohr@informatik.uni-tuebingen.de" - #subject = "Update information for Experiment %s" % expId - #text = "Status of Experiment %s has been updated" % expId #, properties.get("Q_CURRENT_STATUS")) - - #msg = MIMEText(text) - #msg['From'] = fromA - #msg['To'] = toA - #msg['Subject'] = subject - # check for info@qbic.uni-tuebingen.de - #msg['reply-to'] = "mohr@informatik.uni-tuebingen.de" - - #smtpServer = smtplib.SMTP(server) - #smtpServer.sendmail(fromA, toA, msg.as_string()) - #smtpServer.close() - From 46b13be52d0dfaa4f2152551d98587aa3ff201b8 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 12 Jan 2021 14:07:09 +0100 Subject: [PATCH 09/11] Hotfix/wf sample fetching (#58) * convert experiment id to string for v3 objects * adapt changelog * fetch samples by id without using search service --- .../register-16staxonomic-dropbox.py | 10 ++-------- .../register-epitopeprediction.py | 14 +++++--------- .../register-wf-hlatyping/register-hlatyping.py | 11 +++-------- .../register-individualizedproteome-dropbox.py | 14 ++++---------- .../register-ligandomicsid-dropbox.py | 11 ++--------- .../register-ligandomicsqc-dropbox.py | 10 ++-------- .../register-mapping-dropbox.py | 12 +++--------- drop-boxes/register-wf-maqc/register-wf-maqc.py | 12 +++--------- .../register-maxquant-dropbox.py | 12 +++--------- .../register-mergengsdata.py | 11 +++-------- drop-boxes/register-wf-msqc/register-wf-msqc.py | 12 +++--------- drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py | 10 ++-------- drop-boxes/register-wf-peakpicking/script.py | 12 ++++-------- .../register-wf-peptideid/register-wf-peptideid.py | 14 +++----------- drop-boxes/register-wf-qedda/register-qedda.py | 9 ++------- .../register-wf-rnaexpranalysis.py | 11 +++-------- .../register-wf-shrna/register-shrna-dropbox.py | 12 +++--------- .../register-variantannotation.py | 11 +++-------- .../register-wf-variantcalling.py | 13 ++++--------- 19 files changed, 57 insertions(+), 164 deletions(-) diff --git a/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py b/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py index 182a5d7b..ac7ef035 100644 --- a/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py +++ b/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py @@ -56,18 +56,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py b/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py index d8404ba4..8e28dfea 100644 --- a/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py +++ b/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py @@ -34,18 +34,14 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode + if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + sample = transaction.getSampleForUpdate(sample_id) + + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-hlatyping/register-hlatyping.py b/drop-boxes/register-wf-hlatyping/register-hlatyping.py index 3d1b0036..c2e5c1aa 100644 --- a/drop-boxes/register-wf-hlatyping/register-hlatyping.py +++ b/drop-boxes/register-wf-hlatyping/register-hlatyping.py @@ -57,18 +57,13 @@ def process(transaction): project = pPattern.findall(name)[0] experiment_id = ePattern.findall(name)[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py b/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py index 8d61a2f0..f7fbf1dd 100644 --- a/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py +++ b/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode + if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + sample = transaction.getSampleForUpdate(sample_id) + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py b/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py index 98dc78b1..d854ce65 100644 --- a/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py +++ b/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py @@ -41,19 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/" + space + "/" + sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py b/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py index 016b9b08..9ffc8937 100644 --- a/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py +++ b/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py @@ -41,18 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-mapping/register-mapping-dropbox.py b/drop-boxes/register-wf-mapping/register-mapping-dropbox.py index dd87826f..4ca24383 100644 --- a/drop-boxes/register-wf-mapping/register-mapping-dropbox.py +++ b/drop-boxes/register-wf-mapping/register-mapping-dropbox.py @@ -41,22 +41,16 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) additionalInfo = sample.getPropertyValue("Q_ADDITIONAL_INFO") - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-maqc/register-wf-maqc.py b/drop-boxes/register-wf-maqc/register-wf-maqc.py index b6a97400..efe32e30 100644 --- a/drop-boxes/register-wf-maqc/register-wf-maqc.py +++ b/drop-boxes/register-wf-maqc/register-wf-maqc.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() if len(parents) > 6: first = parents[0].split("/")[-1] parentInfos = first+"_and_"+str(len(parents)-1)+"others" diff --git a/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py b/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py index 8c1eff51..9317bde9 100644 --- a/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py +++ b/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py b/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py index 39f10e78..394a44b7 100644 --- a/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py +++ b/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py @@ -34,18 +34,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-msqc/register-wf-msqc.py b/drop-boxes/register-wf-msqc/register-wf-msqc.py index 76ffa18a..ed0e9055 100644 --- a/drop-boxes/register-wf-msqc/register-wf-msqc.py +++ b/drop-boxes/register-wf-msqc/register-wf-msqc.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py b/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py index 6957113a..1fb9643e 100644 --- a/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py +++ b/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py @@ -41,18 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) diff --git a/drop-boxes/register-wf-peakpicking/script.py b/drop-boxes/register-wf-peakpicking/script.py index 03e555ed..ec234c1f 100644 --- a/drop-boxes/register-wf-peakpicking/script.py +++ b/drop-boxes/register-wf-peakpicking/script.py @@ -54,16 +54,12 @@ def process(transaction): #Register logs wfSampleCode = nameSplit[-1] + + sample_id = "/"+space+"/"+wfSampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, wfSampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - wfSample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + wfSample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) @@ -95,4 +91,4 @@ def process(transaction): sa = transaction.getSampleForUpdate(sampleID) dataSetRes = transaction.createNewDataSet('Q_MS_MZML_DATA') dataSetRes.setSample(sa) - transaction.moveFile(mzmlPath, dataSetRes) \ No newline at end of file + transaction.moveFile(mzmlPath, dataSetRes) diff --git a/drop-boxes/register-wf-peptideid/register-wf-peptideid.py b/drop-boxes/register-wf-peptideid/register-wf-peptideid.py index 0bcc3675..60f932bd 100644 --- a/drop-boxes/register-wf-peptideid/register-wf-peptideid.py +++ b/drop-boxes/register-wf-peptideid/register-wf-peptideid.py @@ -41,22 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - #sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.TYPE, "Q_WF_MS_PEPTIDEID_RUN")) - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - #sample = transaction.createNewSample("/"+space+"/"+sample_id + str(len(foundSamples)+1), "Q_WF_MS_PEPTIDEID_RUN") - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-qedda/register-qedda.py b/drop-boxes/register-wf-qedda/register-qedda.py index d5eee34f..5ba864ac 100644 --- a/drop-boxes/register-wf-qedda/register-qedda.py +++ b/drop-boxes/register-wf-qedda/register-qedda.py @@ -34,16 +34,11 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - sample = foundSamples[0] - sample = transaction.getSampleForUpdate(sample.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) diff --git a/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py b/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py index ee4bb2b9..2551720a 100644 --- a/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py +++ b/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py @@ -42,18 +42,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-shrna/register-shrna-dropbox.py b/drop-boxes/register-wf-shrna/register-shrna-dropbox.py index a8d499ea..fde5f461 100644 --- a/drop-boxes/register-wf-shrna/register-shrna-dropbox.py +++ b/drop-boxes/register-wf-shrna/register-shrna-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-variantannotation/register-variantannotation.py b/drop-boxes/register-wf-variantannotation/register-variantannotation.py index 55d13d0c..17d8188a 100644 --- a/drop-boxes/register-wf-variantannotation/register-variantannotation.py +++ b/drop-boxes/register-wf-variantannotation/register-variantannotation.py @@ -34,18 +34,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() #parentcodes = [] #for parent in parents: # parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py b/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py index 2481ff8c..557da3e0 100644 --- a/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py +++ b/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py @@ -42,18 +42,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) @@ -89,4 +84,4 @@ def process(transaction): #if os.path.isdir(incomingPath+"/result"): # transaction.moveFile(incomingPath+"/result", dataSetRes) #else: - # transaction.moveFile(incomingPath, dataSetRes) \ No newline at end of file + # transaction.moveFile(incomingPath, dataSetRes) From 9f554d76d36df65bf12fe643b497b8642cc5c40d Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 22 Jan 2021 13:44:46 +0100 Subject: [PATCH 10/11] Hotfix/retry sample tracking (#59) * retry sample tracking update for incoming data and log failure Co-authored-by: Sven F --- .../register-IdXML-dropbox.py | 15 ++++++++++++- .../register-archived-ms-raw/register-raw.py | 15 ++++++++++++- .../register-archived-ms-raw/register-raw.pyc | Bin 3074 -> 0 bytes .../register-bam-dropbox/register-bam.py | 15 ++++++++++++- .../etl_msconvert.py | 15 +++++++++++-- .../register-fasta-dropbox/register-fasta.py | 15 ++++++++++++- .../register-fastq-with-metadata-dropbox.py | 15 ++++++++++++- .../register-fastq-dropbox.py | 16 ++++++++++++- .../register-fXML-dropbox.py | 15 ++++++++++++- .../register-hlatyping.py | 15 ++++++++++++- .../register-imgag-dropbox/register-imgag.py | 18 ++++++++++++--- .../register-immunmonitoring.py | 15 ++++++++++++- .../register-cel-dropbox.py | 15 ++++++++++++- .../register-mtb-data-dropbox.py | 21 +++++++++++++----- .../register-mzml-dropbox.py | 15 ++++++++++++- .../register-nanopore.py | 15 ++++++++++++- .../register-nmr-dropbox/register-nmr.py | 15 ++++++++++++- .../register-peptidedata.py | 15 ++++++++++++- .../register-qcml-dropbox.py | 15 ++++++++++++- .../register-qpcr-dropbox.py | 16 ++++++++++++- .../register-vcf-dropbox/register-vcf.py | 16 ++++++++++++- .../register-wiff-data-dropbox.py | 15 ++++++++++++- 22 files changed, 299 insertions(+), 28 deletions(-) delete mode 100644 drop-boxes/register-archived-ms-raw/register-raw.pyc diff --git a/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py b/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py index 8fae5fe3..40f8449f 100644 --- a/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py +++ b/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-archived-ms-raw/register-raw.py b/drop-boxes/register-archived-ms-raw/register-raw.py index 9e89bf74..52a7f480 100644 --- a/drop-boxes/register-archived-ms-raw/register-raw.py +++ b/drop-boxes/register-archived-ms-raw/register-raw.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -104,4 +105,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-archived-ms-raw/register-raw.pyc b/drop-boxes/register-archived-ms-raw/register-raw.pyc deleted file mode 100644 index c3375c219b936d41ff02951ec9ab2bb1586c7941..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3074 zcmbVO-E$h(6+a6CAtVGswu$SsA#R(|Lj;MNnKYhEry2>>o($Mtkey(AnAPqzELwK) z-itnxr7y`JkhlCP`IqvZ_qM-t7Lbzkr4`z<_k7%Q&-dNz&!ysDzkJ?zY4MlA`|o)4 zj{qJ~AJ8H%LxDwoi>>MVwWxL z?rpLI&jE_XPGr=>;xzU{Q`g2ZYNA4;>%*vSqKEZ=MpG4@`nsuAe6C{M^c>@86UAzf zQkp?DY+_$*X43)I4RdLV{=f0p(KQ^<97PkgANb)}^R041=i`r>r+(Pnv%oWJ0LOly zQfy;^oftpRDQO#|u5w~`bQt?a#lDl$cUJW7>~zV}?|}O3)x|fT%wK-A|Jmzr{xW~* z?0elHcck1iJ)4*ukYgv%D&-Y{`W_z^xcf1>we=c2h{0BKCGj40Es%c*jW@;J(xNn~ZMQsy3@ILp!C0E@HV<3utiWY1Yn+}1a(Y=5X6r-7@&lyo^)4UcCC?Fr*VOjH4dAX zrMvSSnGH&EjK|;e!ds*IJTI#pR_hd&$P>KgtyAqEkJyx=s4I}!lm#?w(0Gf{af!y; zvM6(Tvu0R;_DTvg-eEbmw@67*&L3Ar!R89z6?{t^zC{W2LmU&BPxNkcoM6KuYwzCl&)bBLh`qNZLh$Aejek!GLM`DNN*7p6{R6>z z@e!KeBd^5Byhcfl#&t69r#v9{$wNFLn?Y)T|B;n0H~xtd6d2+KrgTvS)!9LIiR@O- z{%UaeINcH$DoJ8`m>kZ z_?QK+JPZO%g$MI`6xUY`%7;H2w7Y|L>!5pLx7w{v4-JlqC9owu+U`nMi=2JZF{R}u z`{2dk=%9CCZr!9jZ*`8I*n{3zPg~NYEY*9|vfEGg{tP*D2Vkg-EsY*K#*f0oC^YKQ z7&Zs2rZxN!CY#b^mUg00p%+D{<4n@^ zA|OJ4J)u^Jt^5zxZIsOt}@k|LfCvV#dV}4 zECHg%6EXJX{xW%pI#XO;p5Xxr71!Jw>iHu11+RBCRT9$xIpEl86N_=FvmJGDtw#=C z(NojWjM#%jIx!Ud+8r$w1ojOjocWcKBV4atWkfT?+RD+hSb0pgxg8e4Gf!h&%(1z8 z?gTT1Su1BQEZ~q$qH`rTE8hZ=JQF*ic3iGGDFt^?cv}$g7X`K|s55mX`3g})HiuHS zOyg*bD@RIuD*hcto-$l5X+Gtf097Qtqg}(g-$b*}OD)sV0O`(OM@BA8Em`Ig)Q+5} zE~k|IgLTDi9`DY5g9Sy2pB~YPOmwTU*w~S~atacM~ z8*!SHNSl3-+)+bAR!#I}!}Yb-PzIkW{Ds#@KQ=I&qPXEq{l@tN+3$|zw6SyUH2lbh z4l=g^qId`~2V=E?AvK9Y&20I)d+^^5T6|A7hAKqz+=d>Zki7Jyj~kweWCGdTd2%3> zPVt3k-vvR&mi+}**Xfbf|MvOB{qgxE@@9eh8~2pvmdztA6{IBnaZ2#Zl`|WK{{Yj* B(J%l2 diff --git a/drop-boxes/register-bam-dropbox/register-bam.py b/drop-boxes/register-bam-dropbox/register-bam.py index 186c5004..20a21ad5 100644 --- a/drop-boxes/register-bam-dropbox/register-bam.py +++ b/drop-boxes/register-bam-dropbox/register-bam.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -136,4 +137,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py b/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py index dcb2d9f0..13dd049f 100644 --- a/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py +++ b/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py @@ -484,8 +484,19 @@ def __str__(self): return self.value def handleSampleTracking(barcode): - #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode) + break + except: + print "Updating location for sample "+barcode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def createRawDataSet(transaction, incomingPath, sample, format, time_stamp): rawDataSet = transaction.createNewDataSet("Q_MS_RAW_DATA") diff --git a/drop-boxes/register-fasta-dropbox/register-fasta.py b/drop-boxes/register-fasta-dropbox/register-fasta.py index aaab99a4..2fc5e52a 100644 --- a/drop-boxes/register-fasta-dropbox/register-fasta.py +++ b/drop-boxes/register-fasta-dropbox/register-fasta.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -152,4 +153,16 @@ def process(transaction): transaction.moveFile(new_folder, dataSet) # Updates the sample location of the extract sample - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py b/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py index 6fae6beb..885f1c91 100644 --- a/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py +++ b/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -167,4 +168,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py b/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py index 34d5601f..d89ffec2 100644 --- a/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py +++ b/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -128,4 +129,17 @@ def process(transaction): nameFile.close() transaction.moveFile(incomingPath, dataSet) - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + # sample tracking + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py b/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py index 6c71ba7e..46425e0f 100644 --- a/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py +++ b/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py b/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py index b6fa1ad6..1f213fb6 100644 --- a/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py +++ b/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -149,4 +150,16 @@ def process(transaction): transaction.moveFile(resultPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-imgag-dropbox/register-imgag.py b/drop-boxes/register-imgag-dropbox/register-imgag.py index 02896d4f..6aa33d3f 100644 --- a/drop-boxes/register-imgag-dropbox/register-imgag.py +++ b/drop-boxes/register-imgag-dropbox/register-imgag.py @@ -664,7 +664,19 @@ def process(transaction): transaction.moveFile(vcfFolder, vcfDataSet) else: - find_and_register_ngs_without_metadata(transaction, parentCodes) + find_and_register_ngs_without_metadata(transaction, parentCodes) for code in parentCodes: - #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + #sample tracking section + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + break + except: + print "Updating location for sample "+code+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py b/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py index f4242682..c5757c59 100644 --- a/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py +++ b/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py b/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py index 0c3260db..1da34e40 100644 --- a/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py +++ b/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -192,4 +193,16 @@ def process(transaction): os.remove(os.path.realpath(os.path.join(incomingPath,f))) #sample tracking section for code in trackingCodes: - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + break + except: + print "Updating location for sample "+code+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py b/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py index 43372890..08092b6c 100644 --- a/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py +++ b/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py @@ -164,11 +164,22 @@ ############################################################################# def update_sample_location_to_qbic(sampleId): - """Calls the sample status service and updates the - location to QBiC and the status 'DATA AT QBiC'. - """ - # Update sample location - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId) + """Calls the sample status service and updates the + location to QBiC and the status 'DATA AT QBiC'. + """ + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId) + break + except: + print("Updating location for sample " + sampleId + " failed on attempt "+str(attempt+1)) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def process(transaction): diff --git a/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py b/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py index cd4511ce..a9ae0f71 100644 --- a/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py +++ b/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py @@ -9,6 +9,7 @@ import checksum import re import os +import time import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File from org.apache.commons.io import FileUtils @@ -107,4 +108,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise \ No newline at end of file diff --git a/drop-boxes/register-nanopore-dropbox/register-nanopore.py b/drop-boxes/register-nanopore-dropbox/register-nanopore.py index 7ae690ac..059c4fdd 100644 --- a/drop-boxes/register-nanopore-dropbox/register-nanopore.py +++ b/drop-boxes/register-nanopore-dropbox/register-nanopore.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import shutil from datetime import datetime @@ -291,7 +292,19 @@ def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSam transaction.moveFile(absLogPath, logDataSet) # Updates the sample location of the measured sample - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode) + break + except: + print "Updating location for sample "+parentSampleCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def process(transaction): """Main ETL routine entry point""" diff --git a/drop-boxes/register-nmr-dropbox/register-nmr.py b/drop-boxes/register-nmr-dropbox/register-nmr.py index 262ed122..48192c59 100644 --- a/drop-boxes/register-nmr-dropbox/register-nmr.py +++ b/drop-boxes/register-nmr-dropbox/register-nmr.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import string import os @@ -145,4 +146,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py b/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py index ed881290..5c0867c5 100644 --- a/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py +++ b/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py @@ -9,6 +9,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -86,4 +87,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py b/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py index 7f55f406..f784f479 100644 --- a/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py +++ b/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -104,4 +105,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py b/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py index daea723e..c72e9971 100644 --- a/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py +++ b/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -103,4 +104,17 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise + diff --git a/drop-boxes/register-vcf-dropbox/register-vcf.py b/drop-boxes/register-vcf-dropbox/register-vcf.py index d3c3f21e..5f8aeed3 100644 --- a/drop-boxes/register-vcf-dropbox/register-vcf.py +++ b/drop-boxes/register-vcf-dropbox/register-vcf.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -163,4 +164,17 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise + diff --git a/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py b/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py index 1a1ff5b0..57830bb5 100644 --- a/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py +++ b/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py @@ -1,6 +1,7 @@ from __future__ import print_function import os +import time import re import sys @@ -107,7 +108,19 @@ def register_wiff_pairs(transaction, wiff_pairs, qbic_id): transaction.moveFile(registration_dir, data_set) # Update sample location - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(qbic_id) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(qbic_id) + break + except: + print("Updating location for sample "+qbic_id+" failed on attempt "+str(attempt+1)) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def space_and_project(transaction, qbiccode): From 9e01d1a8e9fce854255924d7c1e2a1898c1ed44b Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 22 Jan 2021 15:59:10 +0100 Subject: [PATCH 11/11] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32e81588..ea336184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog +## 1.6.0 2021-01-22 + +* Fix for workflow result registration: fetch sample by identifier instead of search for robustness against indexing problems +* Retry sample tracking updates twice and log failures that occur + ## 1.5.0 2020-11-03 * New maintenance task: update missing checksum once, after dss starts.