Sync development with master

CHANGELOG.md

@@ -1,8 +1,20 @@
 # Changelog
 
-## Currently in development
 
-* New maintenance task: update missing checksum one, after dss start.
+## 1.6.0 2021-01-22
+
+* Fix for workflow result registration: fetch sample by identifier instead of search for robustness against indexing problems
+* Retry sample tracking updates twice and log failures that occur
+
+## 1.5.0 2020-11-03
+
+* New maintenance task: update missing checksum once, after dss starts.
+* Fix for nanopore registration: rename folders for pooling case
+* Fix for experiment update: force identifier into a string to support v3 API objects
+
+## 1.4.1 2020-11-03
+
+* Imgag dropbox: raise an exception, if files of unknown type are part of the transaction
 
 ## 1.4.0
 

README.md

@@ -1,4 +1,4 @@
-![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/omero-portlet)
+![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/qbicsoftware/etl-scripts)
 ![Python Language](https://img.shields.io/badge/language-python-blue.svg)
 ![License](https://img.shields.io/github/license/qbicsoftware/etl-scripts)
 [![DOI](https://zenodo.org/badge/45912621.svg)](https://zenodo.org/badge/latestdoi/45912621)

drop-boxes/register-IdXML-dropbox/register-IdXML-dropbox.py

@@ -7,6 +7,7 @@
 sys.path.append('/home-link/qeana10/bin/')
 
 import checksum
+import time
 import re
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
@@ -103,4 +104,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise

drop-boxes/register-archived-ms-raw/register-raw.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -104,4 +105,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+                        break
+                except:
+                        print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise

drop-boxes/register-bam-dropbox/register-bam.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -136,4 +137,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+                        break
+                except:
+                        print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise

drop-boxes/register-convert-ms-vendor-format/etl_msconvert.py

@@ -484,8 +484,19 @@ def __str__(self):
         return self.value
 
 def handleSampleTracking(barcode):
-    #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(barcode)
+            break
+        except:
+            print "Updating location for sample "+barcode+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise
 
 def createRawDataSet(transaction, incomingPath, sample, format, time_stamp):
     rawDataSet = transaction.createNewDataSet("Q_MS_RAW_DATA")

drop-boxes/register-fasta-dropbox/register-fasta.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -152,4 +153,16 @@ def process(transaction):
         transaction.moveFile(new_folder, dataSet)
 
         # Updates the sample location of the extract sample
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise

drop-boxes/register-fastq-dropbox-with-metadata/register-fastq-with-metadata-dropbox.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -167,4 +168,16 @@ def process(transaction):
     transaction.moveFile(incomingPath, dataSet)
 
     #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+            break
+        except:
+            print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise

drop-boxes/register-fastq-dropbox/register-fastq-dropbox.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -128,4 +129,17 @@ def process(transaction):
                 nameFile.close()
         transaction.moveFile(incomingPath, dataSet)
 
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        # sample tracking
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise

drop-boxes/register-featureXML-dropbox/register-fXML-dropbox.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -103,4 +104,16 @@ def process(transaction):
         transaction.moveFile(incomingPath, dataSet)
 
         #sample tracking section
-        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+                try:
+                        SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(identifier)
+                        break
+                except:
+                        print "Updating location for sample "+identifier+" failed on attempt "+str(attempt+1)
+                        if attempt < max_attempts -1:
+                                time.sleep(wait_seconds)
+                                continue
+                        else:
+                                raise

drop-boxes/register-hlatyping-dropbox/register-hlatyping.py

@@ -7,6 +7,7 @@
 sys.path.append('/home-link/qeana10/bin/')
 
 import checksum
+import time
 import re
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
@@ -149,4 +150,16 @@ def process(transaction):
     transaction.moveFile(resultPath, dataSet)
 
     #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+            break
+        except:
+            print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise

drop-boxes/register-imgag-dropbox/register-imgag.py

@@ -596,13 +596,14 @@ def process(transaction):
                 print rawFile
                 if rawFile.endswith("vcf") or rawFile.endswith("vcf.gz"):
                     vcfs.append(rawFile)
-                if rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"):
+                elif rawFile.endswith("fastq") or rawFile.endswith("fastq.gz"):
                     fastqs.append(rawFile)
-                if rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"):
+                elif rawFile.endswith("GSvar") or rawFile.endswith("GSvar.gz"):
                     gsvars.append(rawFile)
-                if rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"):
+                elif rawFile.endswith("tsv") or rawFile.endswith("tsv.gz"):
                     tsvs.append(rawFile)
-
+                else:
+                    raise Exception(rawFile + " is of an unsupported format")
 
             #if rawFiles[0].endswith("vcf") or rawFiles[0].endswith("vcf.gz"):
             #	datasetSample = find_and_register_vcf(transaction, jsonContent)
@@ -663,7 +664,19 @@ def process(transaction):
 
             transaction.moveFile(vcfFolder, vcfDataSet)
     else:
-            find_and_register_ngs_without_metadata(transaction, parentCodes)
+        find_and_register_ngs_without_metadata(transaction, parentCodes)
     for code in parentCodes:
-            #sample tracking section
-            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+        #sample tracking section
+        wait_seconds = 1
+        max_attempts = 3
+        for attempt in range(max_attempts):
+            try:
+                SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+                break
+            except:
+                print "Updating location for sample "+code+" failed on attempt "+str(attempt+1)
+                if attempt < max_attempts -1:
+                    time.sleep(wait_seconds)
+                    continue
+                else:
+                    raise

drop-boxes/register-immunmonitoring-dropbox/register-immunmonitoring.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -103,4 +104,16 @@ def process(transaction):
     transaction.moveFile(incomingPath, dataSet)
 
     #sample tracking section
-    SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(parentCode)
+            break
+        except:
+            print "Updating location for sample "+parentCode+" failed on attempt "+str(attempt+1)
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise

drop-boxes/register-microarrays-dropbox/register-cel-dropbox.py

@@ -8,6 +8,7 @@
 
 import checksum
 import re
+import time
 import os
 import ch.systemsx.cisd.etlserver.registrator.api.v2
 from java.io import File
@@ -192,4 +193,16 @@ def process(transaction):
                 os.remove(os.path.realpath(os.path.join(incomingPath,f)))
         #sample tracking section
         for code in trackingCodes:
-                SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+                wait_seconds = 1
+                max_attempts = 3
+                for attempt in range(max_attempts):
+                        try:
+                                SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(code)
+                                break
+                        except:
+                                print "Updating location for sample "+code+" failed on attempt "+str(attempt+1)
+                                if attempt < max_attempts -1:
+                                        time.sleep(wait_seconds)
+                                        continue
+                                else:
+                                        raise

drop-boxes/register-mtb-data-dropbox/register-mtb-data-dropbox.py

@@ -164,11 +164,22 @@
 #############################################################################
 
 def update_sample_location_to_qbic(sampleId):
-   """Calls the sample status service and updates the
-   location to QBiC and the status 'DATA AT QBiC'.
-   """
-   # Update sample location
-   SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId)
+    """Calls the sample status service and updates the
+    location to QBiC and the status 'DATA AT QBiC'.
+    """
+    wait_seconds = 1
+    max_attempts = 3
+    for attempt in range(max_attempts):
+        try:
+            SAMPLE_TRACKER.updateSampleLocationToCurrentLocation(sampleId)
+            break
+        except:
+            print("Updating location for sample " + sampleId + " failed on attempt "+str(attempt+1))
+            if attempt < max_attempts -1:
+                time.sleep(wait_seconds)
+                continue
+            else:
+                raise
 
 
 def process(transaction):