Feature/omero prep release v1

drop-boxes/register-omero-metadata/backendinterface.py

@@ -182,9 +182,7 @@ def register_image_file_with_dataset_id(file_path, dataset_id, usr, pwd, host, p
     ds_id = dataset_id
 
     if ds_id != -1:
-
         cmd = "omero-importer -s " + host + " -p " + str(port) + " -u " + usr + " -w " + pwd + " -d " + str(int(ds_id)) + " " + file_path
-
         proc = subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
@@ -194,18 +192,14 @@ def register_image_file_with_dataset_id(file_path, dataset_id, usr, pwd, host, p
         std_out, std_err = proc.communicate()
 
         if int(proc.returncode) == 0:
-
             for line in std_out.splitlines():
                 if line[:6] == "Image:":
                     image_ids = line[6:].split(',')
                     break
-
         else:
-            image_ids = -1
-
+            image_ids = []
     else:
-        image_ids = -1
-
+        image_ids = []
     return image_ids
 
 
@@ -353,12 +347,18 @@ def add_annotations_to_image(conn, image_id, key_value_data):
 ##app
 
 from optparse import OptionParser
+import ConfigParser
+
+config = ConfigParser.RawConfigParser()
+config.read("imaging_config.properties")
+
 
 ###OMERO server info
-USERNAME = "usr"
-PASSWORD = "pwd"
-HOST = "host"
-PORT = 4064
+USERNAME = config.get('OmeroServerSection', 'omero.username')
+PASSWORD = config.get('OmeroServerSection', 'omero.password')
+HOST = config.get('OmeroServerSection', 'omero.host')
+PORT = int(config.get('OmeroServerSection', 'omero.port'))
+
 
 def get_args():
     parser = OptionParser()
@@ -410,4 +410,4 @@ def get_args():
 
         add_annotations_to_image(conn, str(args.image_id), key_value_data)
 
-        print "annotation done."
+        print "0"

drop-boxes/register-omero-metadata/image_registration_process.py

@@ -7,10 +7,14 @@
 from subprocess import Popen, PIPE
 
 barcode_pattern = re.compile('Q[a-zA-Z0-9]{4}[0-9]{3}[A-Z][a-zA-Z0-9]')
+conda_home_path = "/home/qeana10/miniconda2/"
+omero_lib_path = "/home/qeana10/openbis/servers/core-plugins/QBIC/1/dss/drop-boxes/register-omero-metadata/OMERO.py-5.4.10-ice36-b105"
+etl_home_path = "/home/qeana10/openbis/servers/core-plugins/QBIC/1/dss/drop-boxes/register-omero-metadata/"
+
 
 class ImageRegistrationProcess:
 
-    def __init__(self, transaction, env_name="omero_env_0", project_code="", sample_code=""):
+    def __init__(self, transaction, env_name="omero_env_0", project_code="", sample_code="", conda_path=None, omero_path=None, etl_path=None):
 
         self._transaction = transaction
         self._incoming_file_name = transaction.getIncoming().getName()
@@ -19,19 +23,32 @@ def __init__(self, transaction, env_name="omero_env_0", project_code="", sample_
         self._project_code = project_code
         self._sample_code = sample_code
 
+        ###set env
+        self._conda_path = conda_home_path
+        if not conda_path is None:
+            self._conda_path = conda_path
+
+        self._omero_path = omero_lib_path
+        if not omero_path is None:
+            self._omero_path = omero_path
+
+        self._etl_path= etl_home_path
+        if not etl_path is None:
+            self._etl_path = etl_path
+
         self._init_cmd_list = []
-        self._init_cmd_list.append('eval "$(/home/qeana10/miniconda2/bin/conda shell.bash hook)"')
+        self._init_cmd_list.append('eval "$(' + self._conda_path + 'bin/conda shell.bash hook)"')
         self._init_cmd_list.append('conda activate ' + env_name)
 
-        self._init_cmd_list.append('export OMERO_PREFIX=/home/qeana10/openbis/servers/core-plugins/QBIC/1/dss/drop-boxes/register-omero-metadata/OMERO.py-5.4.10-ice36-b105')
+        self._init_cmd_list.append('export OMERO_PREFIX=' + self._omero_path)
         self._init_cmd_list.append('export PYTHONPATH=$PYTHONPATH:$OMERO_PREFIX/lib/python')
 
         #now use the omero-importer app packaged in the conda env
         #self._init_cmd_list.append('export PATH=$PATH:/home/qeana10/openbis/servers/core-plugins/QBIC/1/dss/drop-boxes/register-omero-metadata/OMERO.server-5.4.10-ice36-b105/bin')
-        self._init_cmd_list.append('export PATH=$PATH:/home/qeana10/miniconda2/envs/' + env_name + '/bin')
+        self._init_cmd_list.append('export PATH=$PATH:' + self._conda_path + 'envs/' + env_name + '/bin')
 
-        #move to the dir where backendinterface.py lives
-        self._init_cmd_list.append('cd /home/qeana10/openbis/servers/core-plugins/QBIC/1/dss/drop-boxes/register-omero-metadata/')
+        #move to the dir where backendinterface.py lives for exec.
+        self._init_cmd_list.append('cd ' + self._etl_path)
 
     def fetchOpenBisSampleCode(self):
         found = barcode_pattern.findall(self._incoming_file_name)
@@ -84,6 +101,10 @@ def requestOmeroDatasetId(self, project_code=None, sample_code=None):
         return ds_id
 
     def registerImageFileInOmero(self, file_path, dataset_id):
+        print "using file_path:"
+        print file_path
+        print "ds id:"
+        print dataset_id
         cmd_list = list(self._init_cmd_list)
         cmd_list.append( "python backendinterface.py -f " + file_path + " -d " + str(dataset_id) )
 
@@ -95,6 +116,9 @@ def registerImageFileInOmero(self, file_path, dataset_id):
         out, err = process.communicate( commands )
 
         id_list = str(out).split()
+        for img_id in id_list:
+            if not img_id.isdigit():
+                return []
 
         return id_list
 

drop-boxes/register-omero-metadata/register-omero.py

@@ -106,7 +106,7 @@ def createNewImagingRun(tr, base_sample, exp, omero_image_ids, run_offset, prope
 	img_run = tr.createNewSample(new_sample_id_with_offset, IMG_RUN_TYPE)
 	img_run.setParentSampleIdentifiers([base_sample.getSampleIdentifier()])
 	img_run.setExperiment(exp)
-	img_run.setPropertyValue(IMG_RUN_OMERO_PROPERTY_CODE, str(omero_image_ids))
+	img_run.setPropertyValue(IMG_RUN_OMERO_PROPERTY_CODE, '\n'.join(omero_image_ids))
 	for incoming_label in sample_property_map:
 		if incoming_label in properties:
 			key = sample_property_map[incoming_label]
@@ -163,8 +163,26 @@ def findMetaDataFile(incomingPath):
 def getPropertyNames(metadataFile):
 	"""Here we could add more complex behaviour later on.
 	"""
+
+	property_names = metadataFile[0].split("\t")
+	for i in range(len(property_names)):
+		property_names[i] = property_names[i].strip().upper()
+
+	return property_names
 
-	return metadataFile[0].split("\t")
+def validatePropertyNames(property_names):
+	"""Validate metadata property names.
+	TODO: call the imaging metadata parser (with json schema).
+	"""
+
+	# fast validation without parser object.
+	required_names = ["IMAGE_FILE_NAME", "IMAGING_MODALITY", "IMAGED_TISSUE", "INSTRUMENT_MANUFACTURER", "INSTRUMENT_USER", "IMAGING_DATE"]
+
+	for name in required_names:
+		if not name in property_names:
+			return False
+
+	return True
 
 def getPropertyMap(line, property_names):
 	"""Build the property map. Here we could add more complex behaviour later on.
@@ -182,6 +200,21 @@ def getPropertyMap(line, property_names):
 
 	return properties
 
+def filterOmeroPropertyMap(property_map):
+	"""Filters map before ingestion into omero server
+	"""
+
+	#the blacklist, e.g. what is going to openBIS or is automatically added to omero (e.g. file name)
+	filter_list = ["IMAGE_FILE_NAME", "INSTRUMENT_USER", "IMAGING_DATE"]
+
+	new_props = {}
+	for key in property_map.keys():
+		if not key in filter_list:
+			new_props[key] = property_map[key]
+
+	return new_props
+
+
 def printPropertyMap(property_map):
 	"""Function to display metadata properties.
 	"""
@@ -203,13 +236,15 @@ def process(transaction):
 
 	# Get the incoming path of the transaction
 	incomingPath = transaction.getIncoming().getAbsolutePath()
+	# Get the name of the incoming folder
+	folderName = transaction.getIncoming().getName()
 
 	print incomingPath
 
 	# 1. Initialize the image registration process
 	registrationProcess = irp.ImageRegistrationProcess(transaction)
 
-	print "started reg process"
+	print "started reg. process"
 
 	# 2. We want to get the openBIS sample code from the incoming data
 	# This tells us to which biological sample the image data was aquired from.
@@ -228,21 +263,32 @@ def process(transaction):
 	# Each dataset in OMERO contains the associated openBIS biological sample id, which
 	# happened during the experimental design registration with the projectwizard.
 
-	print "calling omero"
+	print "calling omero..."
+	#returns -1 if operation failed
 	omero_dataset_id = registrationProcess.requestOmeroDatasetId(project_code=project_code, sample_code=sample_code)
 
+	print "omero dataset id:"
 	print omero_dataset_id
 
+	omero_failed = int(omero_dataset_id) < 0
+	if omero_failed:
+		raise ValueError("Omero did not return expected dataset id.")
+
 	# Find and parse metadata file content
 	metadataFile = findMetaDataFile(incomingPath)
 
+	print "metadataFile:"
 	print metadataFile
 
 	property_names = getPropertyNames(metadataFile)
 
 	print "property names:"
 	print property_names
 
+	valid_names = validatePropertyNames(property_names)
+	if not valid_names:
+		raise ValueError("Invalid Property Names.")
+
 	#keep track of number of images for openBIS ID
 	image_number = 0
 	#Initialize openBIS imaging experiment
@@ -258,8 +304,9 @@ def process(transaction):
 
 		# Retrieve the image file name, please no whitespace characters in filename!
 		fileName = getFileFromLine(line)
-
-		imageFile = os.path.join(incomingPath, fileName)
+		# Due to the datahandler we need to add another subfolder of the same name to the path
+		imageFolder = os.path.join(incomingPath, folderName)
+		imageFile = os.path.join(imageFolder, fileName)
 		print "New incoming image file for OMERO registration:\t" + imageFile
 
 		# 4. After we have received the omero dataset id, we know where to attach the image to
@@ -281,7 +328,7 @@ def process(transaction):
 
 		#one file can have many images, iterate over all img ids
 		for img_id in omero_image_ids:
-			registrationProcess.registerKeyValuePairs(img_id, properties)
+			registrationProcess.registerKeyValuePairs(img_id, filterOmeroPropertyMap(properties))
 
 
 		####
@@ -306,4 +353,4 @@ def process(transaction):
 
 		# 7. Last but not least we create the open science file format for images which is
 		# OMERO-Tiff and store it in OMERO next to the proprierary vendor format.
-		#registrationProcess.triggerOMETiffConversion()
+		#registrationProcess.triggerOMETiffConversion()