diff --git a/CHANGELOG.md b/CHANGELOG.md index ecfb7338..ea336184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,20 @@ # Changelog -## Currently in development -* New maintenance task: update missing checksum one, after dss start. +## 1.6.0 2021-01-22 + +* Fix for workflow result registration: fetch sample by identifier instead of search for robustness against indexing problems +* Retry sample tracking updates twice and log failures that occur + +## 1.5.0 2020-11-03 + +* New maintenance task: update missing checksum once, after dss starts. +* Fix for nanopore registration: rename folders for pooling case +* Fix for experiment update: force identifier into a string to support v3 API objects + +## 1.4.1 2020-11-03 + +* Imgag dropbox: raise an exception, if files of unknown type are part of the transaction ## 1.4.0 diff --git a/README.md b/README.md index 83afc705..a34ef4d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/omero-portlet) +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/etl-scripts) ![Python Language](https://img.shields.io/badge/language-python-blue.svg) ![License](https://img.shields.io/github/license/qbicsoftware/etl-scripts) [![DOI](https://zenodo.org/badge/45912621.svg)](https://zenodo.org/badge/latestdoi/45912621) diff --git a/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py b/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py index 8fae5fe3..40f8449f 100644 --- a/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py +++ b/drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-archived-ms-raw/register-raw.py b/drop-boxes/register-archived-ms-raw/register-raw.py index 9e89bf74..52a7f480 100644 --- a/drop-boxes/register-archived-ms-raw/register-raw.py +++ b/drop-boxes/register-archived-ms-raw/register-raw.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -104,4 +105,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-archived-ms-raw/register-raw.pyc b/drop-boxes/register-archived-ms-raw/register-raw.pyc deleted file mode 100644 index c3375c21..00000000 Binary files a/drop-boxes/register-archived-ms-raw/register-raw.pyc and /dev/null differ diff --git a/drop-boxes/register-bam-dropbox/register-bam.py b/drop-boxes/register-bam-dropbox/register-bam.py index 186c5004..20a21ad5 100644 --- a/drop-boxes/register-bam-dropbox/register-bam.py +++ b/drop-boxes/register-bam-dropbox/register-bam.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -136,4 +137,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py b/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py index dcb2d9f0..13dd049f 100644 --- a/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py +++ b/drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py @@ -484,8 +484,19 @@ def __str__(self): return self.value def handleSampleTracking(barcode): - #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode) + break + except: + print "Updating location for sample "+barcode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def createRawDataSet(transaction, incomingPath, sample, format, time_stamp): rawDataSet = transaction.createNewDataSet("Q_MS_RAW_DATA") diff --git a/drop-boxes/register-fasta-dropbox/register-fasta.py b/drop-boxes/register-fasta-dropbox/register-fasta.py index aaab99a4..2fc5e52a 100644 --- a/drop-boxes/register-fasta-dropbox/register-fasta.py +++ b/drop-boxes/register-fasta-dropbox/register-fasta.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -152,4 +153,16 @@ def process(transaction): transaction.moveFile(new_folder, dataSet) # Updates the sample location of the extract sample - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py b/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py index 6fae6beb..885f1c91 100644 --- a/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py +++ b/drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -167,4 +168,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py b/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py index 34d5601f..d89ffec2 100644 --- a/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py +++ b/drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -128,4 +129,17 @@ def process(transaction): nameFile.close() transaction.moveFile(incomingPath, dataSet) - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + # sample tracking + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py b/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py index 6c71ba7e..46425e0f 100644 --- a/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py +++ b/drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier) + break + except: + print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py b/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py index b6fa1ad6..1f213fb6 100644 --- a/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py +++ b/drop-boxes/register-hlatyping-dropbox/register-hlatyping.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -149,4 +150,16 @@ def process(transaction): transaction.moveFile(resultPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-imgag-dropbox/register-imgag.py b/drop-boxes/register-imgag-dropbox/register-imgag.py index a97ee8e8..6aa33d3f 100644 --- a/drop-boxes/register-imgag-dropbox/register-imgag.py +++ b/drop-boxes/register-imgag-dropbox/register-imgag.py @@ -596,13 +596,14 @@ def process(transaction): print rawFile if rawFile.endswith("vcf") or rawFile.endswith("vcf.gz"): vcfs.append(rawFile) - if rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"): + elif rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"): fastqs.append(rawFile) - if rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"): + elif rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"): gsvars.append(rawFile) - if rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"): + elif rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"): tsvs.append(rawFile) - + else: + raise Exception(rawFile + " is of an unsupported format") #if rawFiles[0].endswith("vcf") or rawFiles[0].endswith("vcf.gz"): # datasetSample = find_and_register_vcf(transaction, jsonContent) @@ -663,7 +664,19 @@ def process(transaction): transaction.moveFile(vcfFolder, vcfDataSet) else: - find_and_register_ngs_without_metadata(transaction, parentCodes) + find_and_register_ngs_without_metadata(transaction, parentCodes) for code in parentCodes: - #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + #sample tracking section + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + break + except: + print "Updating location for sample "+code+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py b/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py index f4242682..c5757c59 100644 --- a/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py +++ b/drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -103,4 +104,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py b/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py index 0c3260db..1da34e40 100644 --- a/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py +++ b/drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -192,4 +193,16 @@ def process(transaction): os.remove(os.path.realpath(os.path.join(incomingPath,f))) #sample tracking section for code in trackingCodes: - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code) + break + except: + print "Updating location for sample "+code+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py b/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py index 43372890..08092b6c 100644 --- a/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py +++ b/drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py @@ -164,11 +164,22 @@ ############################################################################# def update_sample_location_to_qbic(sampleId): - """Calls the sample status service and updates the - location to QBiC and the status 'DATA AT QBiC'. - """ - # Update sample location - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId) + """Calls the sample status service and updates the + location to QBiC and the status 'DATA AT QBiC'. + """ + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId) + break + except: + print("Updating location for sample " + sampleId + " failed on attempt "+str(attempt+1)) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def process(transaction): diff --git a/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py b/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py index cd4511ce..a9ae0f71 100644 --- a/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py +++ b/drop-boxes/register-mzML-dropbox/register-mzml-dropbox.py @@ -9,6 +9,7 @@ import checksum import re import os +import time import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File from org.apache.commons.io import FileUtils @@ -107,4 +108,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise \ No newline at end of file diff --git a/drop-boxes/register-nanopore-dropbox/register-nanopore.py b/drop-boxes/register-nanopore-dropbox/register-nanopore.py index bc6b2a0c..059c4fdd 100644 --- a/drop-boxes/register-nanopore-dropbox/register-nanopore.py +++ b/drop-boxes/register-nanopore-dropbox/register-nanopore.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import shutil from datetime import datetime @@ -226,12 +227,15 @@ def registerUnclassifiedData(transaction, unclassifiedDataMap, runExperiment, cu # moves a subset of nanopore data to a new target path, needed to add fastq and fast5 subfolders to the same dataset def prepareDataFolder(incomingPath, currentPath, destinationPath, dataObject, suffix): name = dataObject.getName() + # if pooled data, folder is named using barcode and needs to be adapted + if not "_" in name: + name = name + "_" + suffix relativePath = dataObject.getRelativePath() # the source path of the currently handled data object (e.g. fast5_fail folder) sourcePath = os.path.join(os.path.dirname(currentPath), relativePath) checksumFile = createChecksumFileForFolder(incomingPath, sourcePath) # destination path containing data type (fastq or fast5), as well as the parent sample code, so pooled samples can be handled - destination = os.path.join(destinationPath, name + "_" + suffix) + destination = os.path.join(destinationPath, name) os.rename(sourcePath, destination) def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSample, openbisExperiment, currentPath, absLogPath): @@ -288,7 +292,19 @@ def createSampleWithData(transaction, space, parentSampleCode, mapWithDataForSam transaction.moveFile(absLogPath, logDataSet) # Updates the sample location of the measured sample - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentSampleCode) + break + except: + print "Updating location for sample "+parentSampleCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def process(transaction): """Main ETL routine entry point""" diff --git a/drop-boxes/register-nmr-dropbox/register-nmr.py b/drop-boxes/register-nmr-dropbox/register-nmr.py index 262ed122..48192c59 100644 --- a/drop-boxes/register-nmr-dropbox/register-nmr.py +++ b/drop-boxes/register-nmr-dropbox/register-nmr.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import string import os @@ -145,4 +146,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py b/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py index ed881290..5c0867c5 100644 --- a/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py +++ b/drop-boxes/register-peptidedata-dropbox/register-peptidedata.py @@ -9,6 +9,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -86,4 +87,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py b/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py index 7f55f406..f784f479 100644 --- a/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py +++ b/drop-boxes/register-qcml-dropbox/register-qcml-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -104,4 +105,16 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise diff --git a/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py b/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py index daea723e..c72e9971 100644 --- a/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py +++ b/drop-boxes/register-qpcr-dropbox/register-qpcr-dropbox.py @@ -7,6 +7,7 @@ sys.path.append('/home-link/qeana10/bin/') import checksum +import time import re import os import ch.systemsx.cisd.etlserver.registrator.api.v2 @@ -103,4 +104,17 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) \ No newline at end of file + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise + diff --git a/drop-boxes/register-vcf-dropbox/register-vcf.py b/drop-boxes/register-vcf-dropbox/register-vcf.py index d3c3f21e..5f8aeed3 100644 --- a/drop-boxes/register-vcf-dropbox/register-vcf.py +++ b/drop-boxes/register-vcf-dropbox/register-vcf.py @@ -8,6 +8,7 @@ import checksum import re +import time import os import ch.systemsx.cisd.etlserver.registrator.api.v2 from java.io import File @@ -163,4 +164,17 @@ def process(transaction): transaction.moveFile(incomingPath, dataSet) #sample tracking section - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode) + break + except: + print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise + diff --git a/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py b/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py index 182a5d7b..ac7ef035 100644 --- a/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py +++ b/drop-boxes/register-wf-16staxonomicprofiling/register-16staxonomic-dropbox.py @@ -56,18 +56,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py b/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py index d8404ba4..8e28dfea 100644 --- a/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py +++ b/drop-boxes/register-wf-epitopeprediction/register-epitopeprediction.py @@ -34,18 +34,14 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode + if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + sample = transaction.getSampleForUpdate(sample_id) + + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-hlatyping/register-hlatyping.py b/drop-boxes/register-wf-hlatyping/register-hlatyping.py index 3d1b0036..c2e5c1aa 100644 --- a/drop-boxes/register-wf-hlatyping/register-hlatyping.py +++ b/drop-boxes/register-wf-hlatyping/register-hlatyping.py @@ -57,18 +57,13 @@ def process(transaction): project = pPattern.findall(name)[0] experiment_id = ePattern.findall(name)[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py b/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py index 8d61a2f0..f7fbf1dd 100644 --- a/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py +++ b/drop-boxes/register-wf-individualizedproteome/register-individualizedproteome-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode + if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + sample = transaction.getSampleForUpdate(sample_id) + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py b/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py index 98dc78b1..d854ce65 100644 --- a/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py +++ b/drop-boxes/register-wf-ligandomicsid/register-ligandomicsid-dropbox.py @@ -41,19 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/" + space + "/" + sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py b/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py index 016b9b08..9ffc8937 100644 --- a/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py +++ b/drop-boxes/register-wf-ligandomicsqc/register-ligandomicsqc-dropbox.py @@ -41,18 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) experiment.setPropertyValue("Q_WF_STATUS", "FINISHED") diff --git a/drop-boxes/register-wf-mapping/register-mapping-dropbox.py b/drop-boxes/register-wf-mapping/register-mapping-dropbox.py index dd87826f..4ca24383 100644 --- a/drop-boxes/register-wf-mapping/register-mapping-dropbox.py +++ b/drop-boxes/register-wf-mapping/register-mapping-dropbox.py @@ -41,22 +41,16 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) additionalInfo = sample.getPropertyValue("Q_ADDITIONAL_INFO") - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-maqc/register-wf-maqc.py b/drop-boxes/register-wf-maqc/register-wf-maqc.py index b6a97400..efe32e30 100644 --- a/drop-boxes/register-wf-maqc/register-wf-maqc.py +++ b/drop-boxes/register-wf-maqc/register-wf-maqc.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() if len(parents) > 6: first = parents[0].split("/")[-1] parentInfos = first+"_and_"+str(len(parents)-1)+"others" diff --git a/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py b/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py index 8c1eff51..9317bde9 100644 --- a/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py +++ b/drop-boxes/register-wf-maxquant/register-maxquant-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py b/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py index 39f10e78..394a44b7 100644 --- a/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py +++ b/drop-boxes/register-wf-mergenngsdata/register-mergengsdata.py @@ -34,18 +34,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-msqc/register-wf-msqc.py b/drop-boxes/register-wf-msqc/register-wf-msqc.py index 76ffa18a..ed0e9055 100644 --- a/drop-boxes/register-wf-msqc/register-wf-msqc.py +++ b/drop-boxes/register-wf-msqc/register-wf-msqc.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py b/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py index 6957113a..1fb9643e 100644 --- a/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py +++ b/drop-boxes/register-wf-ngsqc/register-wf-ngsqc.py @@ -41,18 +41,12 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) diff --git a/drop-boxes/register-wf-peakpicking/script.py b/drop-boxes/register-wf-peakpicking/script.py index 03e555ed..ec234c1f 100644 --- a/drop-boxes/register-wf-peakpicking/script.py +++ b/drop-boxes/register-wf-peakpicking/script.py @@ -54,16 +54,12 @@ def process(transaction): #Register logs wfSampleCode = nameSplit[-1] + + sample_id = "/"+space+"/"+wfSampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, wfSampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - wfSample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) + wfSample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) @@ -95,4 +91,4 @@ def process(transaction): sa = transaction.getSampleForUpdate(sampleID) dataSetRes = transaction.createNewDataSet('Q_MS_MZML_DATA') dataSetRes.setSample(sa) - transaction.moveFile(mzmlPath, dataSetRes) \ No newline at end of file + transaction.moveFile(mzmlPath, dataSetRes) diff --git a/drop-boxes/register-wf-peptideid/register-wf-peptideid.py b/drop-boxes/register-wf-peptideid/register-wf-peptideid.py index 0bcc3675..60f932bd 100644 --- a/drop-boxes/register-wf-peptideid/register-wf-peptideid.py +++ b/drop-boxes/register-wf-peptideid/register-wf-peptideid.py @@ -41,22 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - #sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.TYPE, "Q_WF_MS_PEPTIDEID_RUN")) - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - #sample = transaction.createNewSample("/"+space+"/"+sample_id + str(len(foundSamples)+1), "Q_WF_MS_PEPTIDEID_RUN") - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-qedda/register-qedda.py b/drop-boxes/register-wf-qedda/register-qedda.py index d5eee34f..5ba864ac 100644 --- a/drop-boxes/register-wf-qedda/register-qedda.py +++ b/drop-boxes/register-wf-qedda/register-qedda.py @@ -34,16 +34,11 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() - - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - sample = foundSamples[0] - sample = transaction.getSampleForUpdate(sample.getSampleIdentifier()) + sample = transaction.getSampleForUpdate(sample_id) experiment = transaction.getExperimentForUpdate("/"+space+"/"+project+"/"+experiment_id) diff --git a/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py b/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py index ee4bb2b9..2551720a 100644 --- a/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py +++ b/drop-boxes/register-wf-rnaexpressionanalysis/register-wf-rnaexpranalysis.py @@ -42,18 +42,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-shrna/register-shrna-dropbox.py b/drop-boxes/register-wf-shrna/register-shrna-dropbox.py index a8d499ea..fde5f461 100644 --- a/drop-boxes/register-wf-shrna/register-shrna-dropbox.py +++ b/drop-boxes/register-wf-shrna/register-shrna-dropbox.py @@ -41,20 +41,14 @@ def process(transaction): space = nameSplit[0] project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] - #sample_id = experiment_id+'.' sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-variantannotation/register-variantannotation.py b/drop-boxes/register-wf-variantannotation/register-variantannotation.py index 55d13d0c..17d8188a 100644 --- a/drop-boxes/register-wf-variantannotation/register-variantannotation.py +++ b/drop-boxes/register-wf-variantannotation/register-variantannotation.py @@ -34,18 +34,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() #parentcodes = [] #for parent in parents: # parentcodes.append(parent.split("/")[-1]) diff --git a/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py b/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py index 2481ff8c..557da3e0 100644 --- a/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py +++ b/drop-boxes/register-wf-variantcalling/register-wf-variantcalling.py @@ -42,18 +42,13 @@ def process(transaction): project = pPattern.findall(nameSplit[1])[0] experiment_id = ePattern.findall(nameSplit[2])[0] sampleCode = nameSplit[-1] + sample_id = "/"+space+"/"+sampleCode if not experiment_id: print "The identifier matching the pattern Q\w{4}E\[0-9]+ was not found in the fileName "+name - ss = transaction.getSearchService() + sample = transaction.getSampleForUpdate(sample_id) - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) - foundSamples = ss.searchForSamples(sc) - samplehit = foundSamples[0] - sample = transaction.getSampleForUpdate(samplehit.getSampleIdentifier()) - - parents = samplehit.getParentSampleIdentifiers() + parents = sample.getParentSampleIdentifiers() parentcodes = [] for parent in parents: parentcodes.append(parent.split("/")[-1]) @@ -89,4 +84,4 @@ def process(transaction): #if os.path.isdir(incomingPath+"/result"): # transaction.moveFile(incomingPath+"/result", dataSetRes) #else: - # transaction.moveFile(incomingPath, dataSetRes) \ No newline at end of file + # transaction.moveFile(incomingPath, dataSetRes) diff --git a/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py b/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py index 1a1ff5b0..57830bb5 100644 --- a/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py +++ b/drop-boxes/register-wiff-dropbox/register-wiff-data-dropbox.py @@ -1,6 +1,7 @@ from __future__ import print_function import os +import time import re import sys @@ -107,7 +108,19 @@ def register_wiff_pairs(transaction, wiff_pairs, qbic_id): transaction.moveFile(registration_dir, data_set) # Update sample location - SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(qbic_id) + wait_seconds = 1 + max_attempts = 3 + for attempt in range(max_attempts): + try: + SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(qbic_id) + break + except: + print("Updating location for sample "+qbic_id+" failed on attempt "+str(attempt+1)) + if attempt < max_attempts -1: + time.sleep(wait_seconds) + continue + else: + raise def space_and_project(transaction, qbiccode): diff --git a/maintenance-tasks/checksum-maintenance/plugin.properties b/maintenance-tasks/checksum-maintenance/plugin.properties index 69e8ac15..25de8e99 100644 --- a/maintenance-tasks/checksum-maintenance/plugin.properties +++ b/maintenance-tasks/checksum-maintenance/plugin.properties @@ -1,3 +1,3 @@ # Updates missing checksums class = ch.systemsx.cisd.etlserver.path.PathInfoDatabaseChecksumCalculationTask -execute-only-once = true \ No newline at end of file +execute-only-once = true diff --git a/reporting-plugins/update-experiment-metadata/update.py b/reporting-plugins/update-experiment-metadata/update.py index 5c826561..d39ba2a3 100644 --- a/reporting-plugins/update-experiment-metadata/update.py +++ b/reporting-plugins/update-experiment-metadata/update.py @@ -1,5 +1,3 @@ -import smtplib -from email.mime.text import MIMEText def process(tr, parameters, tableBuilder): """Change properties of experiment @@ -9,30 +7,9 @@ def process(tr, parameters, tableBuilder): if not user == None: tr.setUserId(user) expId = parameters.get("identifier") - exp = tr.getExperimentForUpdate(expId) + exp = tr.getExperimentForUpdate(str(expId)) properties = parameters.get("properties") for prop in properties.keySet(): exp.setPropertyValue(prop, properties.get(prop)) - - #server = "smtpserv.uni-tuebingen.de" - #fromA = "notification_service@qbis.qbic.uni-tuebingen.de" - - # TODO get emails of space users - # Get it via liferay and pass it to this service ? - #toA = "mohr@informatik.uni-tuebingen.de" - #subject = "Update information for Experiment %s" % expId - #text = "Status of Experiment %s has been updated" % expId #, properties.get("Q_CURRENT_STATUS")) - - #msg = MIMEText(text) - #msg['From'] = fromA - #msg['To'] = toA - #msg['Subject'] = subject - # check for info@qbic.uni-tuebingen.de - #msg['reply-to'] = "mohr@informatik.uni-tuebingen.de" - - #smtpServer = smtplib.SMTP(server) - #smtpServer.sendmail(fromA, toA, msg.as_string()) - #smtpServer.close() -