From f70d27f7eaa4c7c74582512f0f170162fc2ae4eb Mon Sep 17 00:00:00 2001 From: Sven F Date: Fri, 22 Jan 2021 16:00:28 +0100 Subject: [PATCH 1/6] Sync development with master (#61) --- CHANGELOG.md | 16 +++++++++-- README.md | 2 +- .../register-IdXML-dropbox.py | 15 +++++++++- .../register-archived-ms-raw/register-raw.py | 15 +++++++++- .../register-archived-ms-raw/register-raw.pyc | Bin 3074 -> 0 bytes .../register-bam-dropbox/register-bam.py | 15 +++++++++- .../etl_msconvert.py | 15 ++++++++-- .../register-fasta-dropbox/register-fasta.py | 15 +++++++++- .../register-fastq-with-metadata-dropbox.py | 15 +++++++++- .../register-fastq-dropbox.py | 16 ++++++++++- .../register-fXML-dropbox.py | 15 +++++++++- .../register-hlatyping.py | 15 +++++++++- .../register-imgag-dropbox/register-imgag.py | 27 +++++++++++++----- .../register-immunmonitoring.py | 15 +++++++++- .../register-cel-dropbox.py | 15 +++++++++- .../register-mtb-data-dropbox.py | 21 ++++++++++---- .../register-mzml-dropbox.py | 15 +++++++++- .../register-nanopore.py | 20 +++++++++++-- .../register-nmr-dropbox/register-nmr.py | 15 +++++++++- .../register-peptidedata.py | 15 +++++++++- .../register-qcml-dropbox.py | 15 +++++++++- .../register-qpcr-dropbox.py | 16 ++++++++++- .../register-vcf-dropbox/register-vcf.py | 16 ++++++++++- .../register-16staxonomic-dropbox.py | 10 ++----- .../register-epitopeprediction.py | 14 ++++----- .../register-hlatyping.py | 11 ++----- ...register-individualizedproteome-dropbox.py | 14 +++------ .../register-ligandomicsid-dropbox.py | 11 ++----- .../register-ligandomicsqc-dropbox.py | 10 ++----- .../register-mapping-dropbox.py | 12 ++------ .../register-wf-maqc/register-wf-maqc.py | 12 ++------ .../register-maxquant-dropbox.py | 12 ++------ .../register-mergengsdata.py | 11 ++----- .../register-wf-msqc/register-wf-msqc.py | 12 ++------ .../register-wf-ngsqc/register-wf-ngsqc.py | 10 ++----- drop-boxes/register-wf-peakpicking/script.py | 12 +++----- .../register-wf-peptideid.py | 14 ++------- .../register-wf-qedda/register-qedda.py | 9 ++---- .../register-wf-rnaexpranalysis.py | 11 ++----- .../register-shrna-dropbox.py | 12 ++------ .../register-variantannotation.py | 11 ++----- .../register-wf-variantcalling.py | 13 +++------ .../register-wiff-data-dropbox.py | 15 +++++++++- .../checksum-maintenance/plugin.properties | 2 +- .../update-experiment-metadata/update.py | 25 +--------------- 45 files changed, 382 insertions(+), 225 deletions(-) delete mode 100644 drop-boxes/register-archived-ms-raw/register-raw.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index ecfb7338..ea336184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,20 @@ # Changelog -## Currently in development -* New maintenance task: update missing checksum one, after dss start. +## 1.6.0 2021-01-22 + +* Fix for workflow result registration: fetch sample by identifier instead of search for robustness against indexing problems +* Retry sample tracking updates twice and log failures that occur + +## 1.5.0 2020-11-03 + +* New maintenance task: update missing checksum once, after dss starts. +* Fix for nanopore registration: rename folders for pooling case +* Fix for experiment update: force identifier into a string to support v3 API objects + +## 1.4.1 2020-11-03 + +* Imgag dropbox: raise an exception, if files of unknown type are part of the transaction ## 1.4.0 diff --git a/README.md b/README.md index 83afc705..a34ef4d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/omero-portlet) +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/etl-scripts) ![Python Language](https://img.shields.io/badge/language-python-blue.svg) ![License](https://img.shields.io/github/license/qbicsoftware/etl-scripts) [![DOI](https://zenodo.org/badge/45912621.svg)](https://zenodo.org/badge/latestdoi/45912621) diff --git a/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py b/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py index 8fae5fe3..40f8449f 100644 --- a/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py +++ b/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-archived-ms-raw/register-raw.py b/drop-boxes/register-archived-ms-raw/register-raw.py index 9e89bf74..52a7f480 100644 --- a/drop-boxes/register-archived-ms-raw/register-raw.py +++ b/drop-boxes/register-archived-ms-raw/register-raw.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -104,4 +105,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-archived-ms-raw/register-raw.pyc b/drop-boxes/register-archived-ms-raw/register-raw.pyc deleted file mode 100644 index c3375c219b936d41ff02951ec9ab2bb1586c7941..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3074 zcmbVO-E$h(6+a6CAtVGswu$SsA#R(|Lj;MNnKYhEry2>>o($Mtkey(AnAPqzELwK) z-itnxr7y`JkhlCP`IqvZ_qM-t7Lbzkr4`z<_k7%Q&-dNz&!ysDzkJ?zY4MlA`|o)4 zj{qJ~AJ8H%LxDwoi>>MVwWxL z?rpLI&jE_XPGr=>;xzU{Q`g2ZYNA4;>%*vSqKEZ=MpG4@`nsuAe6C{M^c>@86UAzf zQkp?DY+_$*X43)I4RdLV{=f0p(KQ^<97PkgANb)}^R041=i`r>r+(Pnv%oWJ0LOly zQfy;^oftpRDQO#|u5w~`bQt?a#lDl$cUJW7>~zV}?|}O3)x|fT%wK-A|Jmzr{xW~* z?0elHcck1iJ)4*ukYgv%D&-Y{`W_z^xcf1>we=c2h{0BKCGj40Es%c*jW@;J(xNn~ZMQsy3@ILp!C0E@HV<3utiWY1Yn+}1a(Y=5X6r-7@&lyo^)4UcCC?Fr*VOjH4dAX zrMvSSnGH&EjK|;e!ds*IJTI#pR_hd&$P>KgtyAqEkJyx=s4I}!lm#?w(0Gf{af!y; zvM6(Tvu0R;_DTvg-eEbmw@67*&L3Ar!R89z6?{t^zC{W2LmU&BPxNkcoM6KuYwzCl&)bBLh`qNZLh$Aejek!GLM`DNN*7p6{R6>z z@e!KeBd^5Byhcfl#&t69r#v9{$wNFLn?Y)T|B;n0H~xtd6d2+KrgTvS)!9LIiR@O- z{%UaeINcH$DoJ8`m>kZ z_?QK+JPZO%g$MI`6xUY`%7;H2w7Y|L>!5pLx7w{v4-JlqC9owu+U`nMi=2JZF{R}u z`{2dk=%9CCZr!9jZ*`8I*n{3zPg~NYEY*9|vfEGg{tP*D2Vkg-EsY*K#*f0oC^YKQ z7&Zs2rZxN!CY#b^mUg00p%+D{<4n@^ zA|OJ4J)u^Jt^5zxZIsOt}@k|LfCvV#dV}4 zECHg%6EXJX{xW%pI#XO;p5Xxr71!Jw>iHu11+RBCRT9$xIpEl86N_=FvmJGDtw#=C z(NojWjM#%jIx!Ud+8r$w1ojOjocWcKBV4atWkfT?+RD+hSb0pgxg8e4Gf!h&%(1z8 z?gTT1Su1BQEZ~q$qH`rTE8hZ=JQF*ic3iGGDFt^?cv}$g7X`K|s55mX`3g})HiuHS zOyg*bD@RIuD*hcto-$l5X+Gtf097Qtqg}(g-$b*}OD)sV0O`(OM@BA8Em`Ig)Q+5} zE~k|IgLTDi9`DY5g9Sy2pB~YPOmwTU*w~S~atacM~ z8*!SHNSl3-+)+bAR!#I}!}Yb-PzIkW{Ds#@KQ=I&qPXEq{l@tN+3$|zw6SyUH2lbh z4l=g^qId`~2V=E?AvK9Y&20I)d+^^5T6|A7hAKqz+=d>Zki7Jyj~kweWCGdTd2%3> zPVt3k-vvR&mi+}**Xfbf|MvOB{qgxE@@9eh8~2pvmdztA6{IBnaZ2#Zl`|WK{{Yj* B(J%l2 diff --git a/drop-boxes/register-bam-dropbox/register-bam.py b/drop-boxes/register-bam-dropbox/register-bam.py index 186c5004..20a21ad5 100644 --- a/drop-boxes/register-bam-dropbox/register-bam.py +++ b/drop-boxes/register-bam-dropbox/register-bam.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -136,4 +137,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py b/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py index dcb2d9f0..13dd049f 100644 --- a/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py +++ b/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py @@ -484,8 +484,19 @@ def __str__(self): return self.value def handleSampleTracking(barcode): - #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode) + break + except: + print "Updating location for sample "+barcode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def createRawDataSet(transaction, incomingPath, sample, format, time_stamp): rawDataSet = transaction.createNewDataSet("Q_MS_RAW_DATA") diff --git a/drop-boxes/register-fasta-dropbox/register-fasta.py b/drop-boxes/register-fasta-dropbox/register-fasta.py index aaab99a4..2fc5e52a 100644 --- a/drop-boxes/register-fasta-dropbox/register-fasta.py +++ b/drop-boxes/register-fasta-dropbox/register-fasta.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -152,4 +153,16 @@ def process(transaction): transaction.moveFile(new_folder, dataSet) # Updates the sample location of the extract sample - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py b/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py index 6fae6beb..885f1c91 100644 --- a/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py +++ b/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -167,4 +168,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py b/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py index 34d5601f..d89ffec2 100644 --- a/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py +++ b/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -128,4 +129,17 @@ def process(transaction): nameFile.close() transaction.moveFile(incomingPath, dataSet) - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + # sample tracking + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py b/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py index 6c71ba7e..46425e0f 100644 --- a/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py +++ b/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py b/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py index b6fa1ad6..1f213fb6 100644 --- a/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py +++ b/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -149,4 +150,16 @@ def process(transaction): transaction.moveFile(resultPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-imgag-dropbox/register-imgag.py b/drop-boxes/register-imgag-dropbox/register-imgag.py index a97ee8e8..6aa33d3f 100644 --- a/drop-boxes/register-imgag-dropbox/register-imgag.py +++ b/drop-boxes/register-imgag-dropbox/register-imgag.py @@ -596,13 +596,14 @@ def process(transaction): print rawFile if rawFile.endswith("vcf") or rawFile.endswith("vcf.gz"): vcfs.append(rawFile) - if rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"): + elif rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"): fastqs.append(rawFile) - if rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"): + elif rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"): gsvars.append(rawFile) - if rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"): + elif rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"): tsvs.append(rawFile) - + else: + raise Exception(rawFile + " is of an unsupported format") #if rawFiles[0].endswith("vcf") or rawFiles[0].endswith("vcf.gz"): # datasetSample = find_and_register_vcf(transaction, jsonContent) @@ -663,7 +664,19 @@ def process(transaction): transaction.moveFile(vcfFolder, vcfDataSet) else: - find_and_register_ngs_without_metadata(transaction, parentCodes) + find_and_register_ngs_without_metadata(transaction, parentCodes) for code in parentCodes: - #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + #sample tracking section + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + break + except: + print "Updating location for sample "+code+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py b/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py index f4242682..c5757c59 100644 --- a/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py +++ b/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py b/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py index 0c3260db..1da34e40 100644 --- a/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py +++ b/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -192,4 +193,16 @@ def process(transaction): os.remove(os.path.realpath(os.path.join(incomingPath,f))) #sample tracking section for code in trackingCodes: - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + break + except: + print "Updating location for sample "+code+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py b/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py index 43372890..08092b6c 100644 --- a/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py +++ b/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py @@ -164,11 +164,22 @@ ############################################################################# def update_sample_location_to_qbic(sampleId): - """Calls the sample status service and updates the - location to QBiC and the status 'DATA AT QBiC'. - """ - # Update sample location - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId) + """Calls the sample status service and updates the + location to QBiC and the status 'DATA AT QBiC'. + """ + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId) + break + except: + print("Updating location for sample " + sampleId + " failed on attempt "+str(attempt+1)) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def process(transaction): diff --git a/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py b/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py index cd4511ce..a9ae0f71 100644 --- a/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py +++ b/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py @@ -9,6 +9,7 @@ import checksum import re import os +import time import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File from org.apache.commons.io import FileUtils @@ -107,4 +108,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise \ No newline at end of file diff --git a/drop-boxes/register-nanopore-dropbox/register-nanopore.py b/drop-boxes/register-nanopore-dropbox/register-nanopore.py index bc6b2a0c..059c4fdd 100644 --- a/drop-boxes/register-nanopore-dropbox/register-nanopore.py +++ b/drop-boxes/register-nanopore-dropbox/register-nanopore.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import shutil from datetime import datetime @@ -226,12 +227,15 @@ def registerUnclassifiedData(transaction, unclassifiedDataMap, runExperiment, cu # moves a subset of nanopore data to a new target path, needed to add fastq and fast5 subfolders to the same dataset def prepareDataFolder(incomingPath, currentPath, destinationPath, dataObject, suffix): name = dataObject.getName() + # if pooled data, folder is named using barcode and needs to be adapted + if not "_" in name: + name = name + "_" + suffix relativePath = dataObject.getRelativePath() # the source path of the currently handled data object (e.g. fast5_fail folder) sourcePath = os.path.join(os.path.dirname(currentPath), relativePath) checksumFile = createChecksumFileForFolder(incomingPath, sourcePath) # destination path containing data type (fastq or fast5), as well as the parent sample code, so pooled samples can be handled - destination = os.path.join(destinationPath, name + "_" + suffix) + destination = os.path.join(destinationPath, name) os.rename(sourcePath, destination) def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSample, openbisExperiment, currentPath, absLogPath): @@ -288,7 +292,19 @@ def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSam transaction.moveFile(absLogPath, logDataSet) # Updates the sample location of the measured sample - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode) + break + except: + print "Updating location for sample "+parentSampleCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def process(transaction): """Main ETL routine entry point""" diff --git a/drop-boxes/register-nmr-dropbox/register-nmr.py b/drop-boxes/register-nmr-dropbox/register-nmr.py index 262ed122..48192c59 100644 --- a/drop-boxes/register-nmr-dropbox/register-nmr.py +++ b/drop-boxes/register-nmr-dropbox/register-nmr.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import string import os @@ -145,4 +146,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py b/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py index ed881290..5c0867c5 100644 --- a/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py +++ b/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py @@ -9,6 +9,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -86,4 +87,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py b/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py index 7f55f406..f784f479 100644 --- a/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py +++ b/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -104,4 +105,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py b/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py index daea723e..c72e9971 100644 --- a/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py +++ b/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -103,4 +104,17 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise + diff --git a/drop-boxes/register-vcf-dropbox/register-vcf.py b/drop-boxes/register-vcf-dropbox/register-vcf.py index d3c3f21e..5f8aeed3 100644 --- a/drop-boxes/register-vcf-dropbox/register-vcf.py +++ b/drop-boxes/register-vcf-dropbox/register-vcf.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -163,4 +164,17 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise + diff --git a/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py b/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py index 182a5d7b..ac7ef035 100644 --- a/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py +++ b/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py @@ -56,18 +56,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py b/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py index d8404ba4..8e28dfea 100644 --- a/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py +++ b/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py @@ -34,18 +34,14 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode + if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + sample = transaction.getSampleForUpdate(sample_id) + + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-hlatyping/register-hlatyping.py b/drop-boxes/register-wf-hlatyping/register-hlatyping.py index 3d1b0036..c2e5c1aa 100644 --- a/drop-boxes/register-wf-hlatyping/register-hlatyping.py +++ b/drop-boxes/register-wf-hlatyping/register-hlatyping.py @@ -57,18 +57,13 @@ def process(transaction): project = pPattern.findall(name)[0] experiment_id = ePattern.findall(name)[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py b/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py index 8d61a2f0..f7fbf1dd 100644 --- a/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py +++ b/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode + if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + sample = transaction.getSampleForUpdate(sample_id) + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py b/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py index 98dc78b1..d854ce65 100644 --- a/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py +++ b/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py @@ -41,19 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/" + space + "/" + sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py b/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py index 016b9b08..9ffc8937 100644 --- a/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py +++ b/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py @@ -41,18 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-mapping/register-mapping-dropbox.py b/drop-boxes/register-wf-mapping/register-mapping-dropbox.py index dd87826f..4ca24383 100644 --- a/drop-boxes/register-wf-mapping/register-mapping-dropbox.py +++ b/drop-boxes/register-wf-mapping/register-mapping-dropbox.py @@ -41,22 +41,16 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) additionalInfo = sample.getPropertyValue("Q_ADDITIONAL_INFO") - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-maqc/register-wf-maqc.py b/drop-boxes/register-wf-maqc/register-wf-maqc.py index b6a97400..efe32e30 100644 --- a/drop-boxes/register-wf-maqc/register-wf-maqc.py +++ b/drop-boxes/register-wf-maqc/register-wf-maqc.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() if len(parents) > 6: first = parents[0].split("/")[-1] parentInfos = first+"_and_"+str(len(parents)-1)+"others" diff --git a/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py b/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py index 8c1eff51..9317bde9 100644 --- a/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py +++ b/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py b/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py index 39f10e78..394a44b7 100644 --- a/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py +++ b/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py @@ -34,18 +34,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-msqc/register-wf-msqc.py b/drop-boxes/register-wf-msqc/register-wf-msqc.py index 76ffa18a..ed0e9055 100644 --- a/drop-boxes/register-wf-msqc/register-wf-msqc.py +++ b/drop-boxes/register-wf-msqc/register-wf-msqc.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py b/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py index 6957113a..1fb9643e 100644 --- a/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py +++ b/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py @@ -41,18 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) diff --git a/drop-boxes/register-wf-peakpicking/script.py b/drop-boxes/register-wf-peakpicking/script.py index 03e555ed..ec234c1f 100644 --- a/drop-boxes/register-wf-peakpicking/script.py +++ b/drop-boxes/register-wf-peakpicking/script.py @@ -54,16 +54,12 @@ def process(transaction): #Register logs wfSampleCode = nameSplit[-1] + + sample_id = "/"+space+"/"+wfSampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, wfSampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - wfSample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + wfSample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) @@ -95,4 +91,4 @@ def process(transaction): sa = transaction.getSampleForUpdate(sampleID) dataSetRes = transaction.createNewDataSet('Q_MS_MZML_DATA') dataSetRes.setSample(sa) - transaction.moveFile(mzmlPath, dataSetRes) \ No newline at end of file + transaction.moveFile(mzmlPath, dataSetRes) diff --git a/drop-boxes/register-wf-peptideid/register-wf-peptideid.py b/drop-boxes/register-wf-peptideid/register-wf-peptideid.py index 0bcc3675..60f932bd 100644 --- a/drop-boxes/register-wf-peptideid/register-wf-peptideid.py +++ b/drop-boxes/register-wf-peptideid/register-wf-peptideid.py @@ -41,22 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - #sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.TYPE, "Q_WF_MS_PEPTIDEID_RUN")) - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - #sample = transaction.createNewSample("/"+space+"/"+sample_id + str(len(foundSamples)+1), "Q_WF_MS_PEPTIDEID_RUN") - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-qedda/register-qedda.py b/drop-boxes/register-wf-qedda/register-qedda.py index d5eee34f..5ba864ac 100644 --- a/drop-boxes/register-wf-qedda/register-qedda.py +++ b/drop-boxes/register-wf-qedda/register-qedda.py @@ -34,16 +34,11 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - sample = foundSamples[0] - sample = transaction.getSampleForUpdate(sample.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) diff --git a/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py b/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py index ee4bb2b9..2551720a 100644 --- a/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py +++ b/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py @@ -42,18 +42,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-shrna/register-shrna-dropbox.py b/drop-boxes/register-wf-shrna/register-shrna-dropbox.py index a8d499ea..fde5f461 100644 --- a/drop-boxes/register-wf-shrna/register-shrna-dropbox.py +++ b/drop-boxes/register-wf-shrna/register-shrna-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-variantannotation/register-variantannotation.py b/drop-boxes/register-wf-variantannotation/register-variantannotation.py index 55d13d0c..17d8188a 100644 --- a/drop-boxes/register-wf-variantannotation/register-variantannotation.py +++ b/drop-boxes/register-wf-variantannotation/register-variantannotation.py @@ -34,18 +34,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() #parentcodes = [] #for parent in parents: # parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py b/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py index 2481ff8c..557da3e0 100644 --- a/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py +++ b/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py @@ -42,18 +42,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) @@ -89,4 +84,4 @@ def process(transaction): #if os.path.isdir(incomingPath+"/result"): # transaction.moveFile(incomingPath+"/result", dataSetRes) #else: - # transaction.moveFile(incomingPath, dataSetRes) \ No newline at end of file + # transaction.moveFile(incomingPath, dataSetRes) diff --git a/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py b/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py index 1a1ff5b0..57830bb5 100644 --- a/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py +++ b/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py @@ -1,6 +1,7 @@ from __future__ import print_function import os +import time import re import sys @@ -107,7 +108,19 @@ def register_wiff_pairs(transaction, wiff_pairs, qbic_id): transaction.moveFile(registration_dir, data_set) # Update sample location - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(qbic_id) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(qbic_id) + break + except: + print("Updating location for sample "+qbic_id+" failed on attempt "+str(attempt+1)) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def space_and_project(transaction, qbiccode): diff --git a/maintenance-tasks/checksum-maintenance/plugin.properties b/maintenance-tasks/checksum-maintenance/plugin.properties index 69e8ac15..25de8e99 100644 --- a/maintenance-tasks/checksum-maintenance/plugin.properties +++ b/maintenance-tasks/checksum-maintenance/plugin.properties @@ -1,3 +1,3 @@ # Updates missing checksums class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseChecksumCalculationTask -execute-only-once = true \ No newline at end of file +execute-only-once = true diff --git a/reporting-plugins/update-experiment-metadata/update.py b/reporting-plugins/update-experiment-metadata/update.py index 5c826561..d39ba2a3 100644 --- a/reporting-plugins/update-experiment-metadata/update.py +++ b/reporting-plugins/update-experiment-metadata/update.py @@ -1,5 +1,3 @@ -import smtplib -from email.mime.text import MIMEText def process(tr, parameters, tableBuilder): """Change properties of experiment @@ -9,30 +7,9 @@ def process(tr, parameters, tableBuilder): if not user == None: tr.setUserId(user) expId = parameters.get("identifier") - exp = tr.getExperimentForUpdate(expId) + exp = tr.getExperimentForUpdate(str(expId)) properties = parameters.get("properties") for prop in properties.keySet(): exp.setPropertyValue(prop, properties.get(prop)) - - #server = "smtpserv.uni-tuebingen.de" - #fromA = "notification_service@qbis.qbic.uni-tuebingen.de" - - # TODO get emails of space users - # Get it via liferay and pass it to this service ? - #toA = "mohr@informatik.uni-tuebingen.de" - #subject = "Update information for Experiment %s" % expId - #text = "Status of Experiment %s has been updated" % expId #, properties.get("Q_CURRENT_STATUS")) - - #msg = MIMEText(text) - #msg['From'] = fromA - #msg['To'] = toA - #msg['Subject'] = subject - # check for info@qbic.uni-tuebingen.de - #msg['reply-to'] = "mohr@informatik.uni-tuebingen.de" - - #smtpServer = smtplib.SMTP(server) - #smtpServer.sendmail(fromA, toA, msg.as_string()) - #smtpServer.close() - From 7702b394687c2e24c47f021765b2cefb8e4a369c Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 22 Feb 2021 17:20:38 +0100 Subject: [PATCH 2/6] adapt to version running in production attachments server --- .../register-attachment-dropbox.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py b/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py index cd82f564..ae4615b2 100755 --- a/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py +++ b/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py @@ -84,26 +84,14 @@ def process(transaction): sa = transaction.getSampleForUpdate(sampleID) space = sa.getSpace() if not attachmentReady: - expID = '/' + space + '/' + project + '/'+ project+'_INFO' - exp = transaction.getExperimentForUpdate(expID) - if not exp: - exp = transaction.createNewExperiment(expID, "Q_PROJECT_DETAILS") + infoSampleID = "/"+space+"/"+code + sa = transaction.getSampleForUpdate(infoSampleID) + if not sa: + exp = transaction.createNewExperiment('/' + space + '/' + project + '/'+ project+'_INFO', "Q_PROJECT_DETAILS") sa = transaction.createNewSample('/' + space + '/'+ code, "Q_ATTACHMENT_SAMPLE") sa.setExperiment(exp) info = None - #if isProject: - #experiments = search_service.listExperiments("/" + space + "/" + project) - #for e in experiments: - # if project+"_INFO" in e.getExperimentIdentifier(): - # info = e - #if not info: - # info = transaction.createNewExperiment('/' + space + '/' + project + '/'+ project+'_INFO', "Q_PROJECT_DETAILS") - #else: - # info = transaction.getExperiment('/' + space + '/' + project + '/' + code) - # register new experiment and sample - #sa.setExperiment(info) - # create new dataset dataSet = transaction.createNewDataSet("Q_PROJECT_DATA") dataSet.setMeasuredData(False) dataSet.setPropertyValue("Q_SECONDARY_NAME", secname) From 32092ebc0a4e0306e1432e38311ac12ae6bf4b03 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 22 Feb 2021 17:37:37 +0100 Subject: [PATCH 3/6] add readme --- .../register-attachments-dropbox/README.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 drop-boxes/register-attachments-dropbox/README.md diff --git a/drop-boxes/register-attachments-dropbox/README.md b/drop-boxes/register-attachments-dropbox/README.md new file mode 100644 index 00000000..450fdec0 --- /dev/null +++ b/drop-boxes/register-attachments-dropbox/README.md @@ -0,0 +1,33 @@ +# Attachments Dropbox + +## Expected data structure +The data structure needs to be a root folder, containing a file `metadata.txt`. + +Incoming structure overview: + +``` +|- (top level folder name, normally a time stamp of upload time) + | + |- metadata.txt +``` + +openBIS structure overview: + +Attachments are attached to the Q_PROJECT_DETAILS experiment type and its sample type Q_ATTACHMENT_SAMPLE. + +TODO: ER model. + +## Expected metadata +Metadata is expected to be denoted in line-separated key-value pairs, where key and value are separated by a '='. The following structure/pairs are expected: + +``` +user= +info= +barcode= +type= +``` +The code of the attachment sample is built from the project code followed by three zeroes, e.g. QABCD000. + +See code examples: +https://github.com/qbicsoftware/attachi-cli/blob/master/attachi/attachi.py#L63 +https://github.com/qbicsoftware/projectwizard-portlet/blob/9c86f500b26af4cf2613cfae32e470bf5d50bf78/src/main/java/life/qbic/projectwizard/io/AttachmentMover.java#L145 From bcf1c5a677ddc37f3e83b179dd1c8a96631000e7 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Feb 2021 16:35:34 +0100 Subject: [PATCH 4/6] revert change that was not part of this feature --- .../register-attachment-dropbox.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py b/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py index ae4615b2..27abd34c 100755 --- a/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py +++ b/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py @@ -84,10 +84,10 @@ def process(transaction): sa = transaction.getSampleForUpdate(sampleID) space = sa.getSpace() if not attachmentReady: - infoSampleID = "/"+space+"/"+code - sa = transaction.getSampleForUpdate(infoSampleID) - if not sa: - exp = transaction.createNewExperiment('/' + space + '/' + project + '/'+ project+'_INFO', "Q_PROJECT_DETAILS") + expID = '/' + space + '/' + project + '/'+ project+'_INFO' + exp = transaction.getExperimentForUpdate(expID) + if not exp: + exp = transaction.createNewExperiment(expID, "Q_PROJECT_DETAILS") sa = transaction.createNewSample('/' + space + '/'+ code, "Q_ATTACHMENT_SAMPLE") sa.setExperiment(exp) info = None From 22e4b2097a22b57b4103371256d860959507244a Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Feb 2021 17:56:07 +0100 Subject: [PATCH 5/6] add regex description --- drop-boxes/register-attachments-dropbox/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drop-boxes/register-attachments-dropbox/README.md b/drop-boxes/register-attachments-dropbox/README.md index 450fdec0..18a2fc1a 100644 --- a/drop-boxes/register-attachments-dropbox/README.md +++ b/drop-boxes/register-attachments-dropbox/README.md @@ -26,7 +26,7 @@ info= barcode= type= ``` -The code of the attachment sample is built from the project code followed by three zeroes, e.g. QABCD000. +The code of the attachment sample is built from the project code followed by three zeroes, conforming to the regular expression "Q[A-Z0-9]{4}000", e.g. QABCD000. See code examples: https://github.com/qbicsoftware/attachi-cli/blob/master/attachi/attachi.py#L63 From b1d0d8258ccaacf6855dd3e7286b55f093cf1247 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 26 Feb 2021 14:28:23 +0100 Subject: [PATCH 6/6] include attachment readme in main readme --- README.md | 35 +++++++++++++++++++ .../register-attachments-dropbox/README.md | 33 ----------------- 2 files changed, 35 insertions(+), 33 deletions(-) delete mode 100644 drop-boxes/register-attachments-dropbox/README.md diff --git a/README.md b/README.md index a34ef4d3..8b3a70a3 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,38 @@ look like this: ``` +### Attachment Data + +**Responsible dropbox:** +[QBiC-register-exp-proj-attachment](drop-boxes/register-attachments-dropbox) + +**openBIS structure:** + +Attachments are attached to the Q_PROJECT_DETAILS experiment type and its sample type Q_ATTACHMENT_SAMPLE. + +**Expected data structure** +The data structure needs to be a root folder, containing a file `metadata.txt`. + +Incoming structure overview: + +``` +|- (top level folder name, normally a time stamp of upload time) + | + |- metadata.txt +``` + +**Expected metadata** +Metadata is expected to be denoted in line-separated key-value pairs, where key and value are separated by a '='. The following structure/pairs are expected: + +``` +user= +info= +barcode= +type= +``` +The code of the attachment sample is built from the project code followed by three zeroes, conforming to the regular expression "Q[A-Z0-9]{4}000", e.g. QABCD000. + +See code examples: +https://github.com/qbicsoftware/attachi-cli/blob/master/attachi/attachi.py#L63 +https://github.com/qbicsoftware/projectwizard-portlet/blob/9c86f500b26af4cf2613cfae32e470bf5d50bf78/src/main/java/life/qbic/projectwizard/io/AttachmentMover.java#L145 + diff --git a/drop-boxes/register-attachments-dropbox/README.md b/drop-boxes/register-attachments-dropbox/README.md deleted file mode 100644 index 18a2fc1a..00000000 --- a/drop-boxes/register-attachments-dropbox/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Attachments Dropbox - -## Expected data structure -The data structure needs to be a root folder, containing a file `metadata.txt`. - -Incoming structure overview: - -``` -|- (top level folder name, normally a time stamp of upload time) - | - |- metadata.txt -``` - -openBIS structure overview: - -Attachments are attached to the Q_PROJECT_DETAILS experiment type and its sample type Q_ATTACHMENT_SAMPLE. - -TODO: ER model. - -## Expected metadata -Metadata is expected to be denoted in line-separated key-value pairs, where key and value are separated by a '='. The following structure/pairs are expected: - -``` -user= -info= -barcode= -type= -``` -The code of the attachment sample is built from the project code followed by three zeroes, conforming to the regular expression "Q[A-Z0-9]{4}000", e.g. QABCD000. - -See code examples: -https://github.com/qbicsoftware/attachi-cli/blob/master/attachi/attachi.py#L63 -https://github.com/qbicsoftware/projectwizard-portlet/blob/9c86f500b26af4cf2613cfae32e470bf5d50bf78/src/main/java/life/qbic/projectwizard/io/AttachmentMover.java#L145