Deployment prep

flir/Dockerfile

@@ -35,4 +35,4 @@ CMD ["extractor"]
 ENV RABBITMQ_EXCHANGE="terra" \
     RABBITMQ_VHOST="%2F" \
     RABBITMQ_QUEUE="terra.flirExtractor" \
-    MAIN_SCRIPT="terra_flirExtractor.py"
+    MAIN_SCRIPT="terra.flir.py"

flir/Get_FLIR.py

@@ -96,26 +96,31 @@ def get_flir(in_dir, out_dir):
             os.mkdir(out_dir)
         except:
             fail('Failed to create directory in ' + out_dir)
-
-    metadata = lower_keys(load_json(metafile)) # load json file
-
-    center_position, scan_time, fov = parse_metadata(metadata) # get information from json file
-
-    gps_bounds = get_bounding_box(center_position, fov) # get bounding box using gantry position and fov of camera
-
+
+
     raw_data = load_flir_data(binfile) # get raw data from bin file
 
     filename = os.path.basename(binfile)
     temp_name = os.path.join(out_dir, filename)
     out_png = temp_name[:-3] + 'png'
 
     im_color = create_png(raw_data, out_png) # create png
-
-    tc = rawData_to_temperature(raw_data, scan_time, metadata) # get temperature
-
-    tif_path = temp_name[:-3] + 'tif'
-
-    create_geotiff_with_temperature(im_color, tc, gps_bounds, tif_path) # create geotiff
+
+
+    metadata = lower_keys(load_json(metafile)) # load json file
+
+    center_position, scan_time, fov = parse_metadata(metadata) # get information from json file
+
+    if center_position is None or scan_time is None or fov is None:
+        print("error getting metadata; skipping geoTIFF")
+    else:
+        gps_bounds = get_bounding_box(center_position, fov) # get bounding box using gantry position and fov of camera
+
+        tc = rawData_to_temperature(raw_data, scan_time, metadata) # get temperature
+
+        tif_path = temp_name[:-3] + 'tif'
+
+        create_geotiff_with_temperature(im_color, tc, gps_bounds, tif_path) # create geotiff
 
     return
 
@@ -360,14 +365,16 @@ def parse_metadata(metadata):
         else:
             cam_z = 0
 
+        position = [float(gantry_x), float(gantry_y), float(gantry_z)]
+        center_position = [position[0]+float(cam_x), position[1]+float(cam_y), position[2]+float(cam_z)]
+        fov = [float(fov_x), float(fov_y)]
+
+        return center_position, scan_time, fov
+
     except KeyError as err:
         fail('Metadata file missing key: ' + err.args[0])
-
-    position = [float(gantry_x), float(gantry_y), float(gantry_z)]
-    center_position = [position[0]+float(cam_x), position[1]+float(cam_y), position[2]+float(cam_z)]
-    fov = [float(fov_x), float(fov_y)]
-
-    return center_position, scan_time, fov
+        return None, None, None
+
 
 def lower_keys(in_dict):
     if type(in_dict) is dict:

flir/terra_flirExtractor.py → flir/terra.flir.py

@@ -40,31 +40,27 @@ def main():
 def check_message(parameters):
     # Check for a left and right file before beginning processing
     found_ir = False
-    found_json = False
+    found_md = False
     for f in parameters['filelist']:
         if 'filename' in f and f['filename'].endswith('_ir.bin'):
             found_ir = True
         elif 'filename' in f and f['filename'].endswith('_metadata.json'):
-            found_json = True
-
-    if not (found_ir and found_json):
-        return False
-
-    # TODO: re-enable once this is merged into Clowder: https://opensource.ncsa.illinois.edu/bitbucket/projects/CATS/repos/clowder/pull-requests/883/overview
-    # fetch metadata from dataset to check if we should remove existing entry for this extractor first
-    md = extractors.download_dataset_metadata_jsonld(parameters['host'], parameters['secretKey'], parameters['datasetId'], extractorName)
-    found_meta = False
-    for m in md:
-        if 'agent' in m and 'name' in m['agent']:
-            if m['agent']['name'].find(extractorName) > -1:
-                print("skipping dataset %s, already processed" % parameters['datasetId'])
-                return False
-                #extractors.remove_dataset_metadata_jsonld(parameters['host'], parameters['secretKey'], parameters['datasetId'], extractorName)
-        # Check for required metadata before beginning processing
-        if 'content' in m and 'lemnatec_measurement_metadata' in m['content']:
-            found_meta = True
-
-    if found_ir and found_json:
+            found_md = True
+
+    # If we don't find _metadata.json file, check if we have metadata attached to dataset instead
+    if not found_md:
+        md = extractors.download_dataset_metadata_jsonld(parameters['host'], parameters['secretKey'], parameters['datasetId'], extractorName)
+        if len(md) > 0:
+            for m in md:
+                # Check if this extractor has already been processed
+                if 'agent' in m and 'name' in m['agent']:
+                    if m['agent']['name'].find(extractorName) > -1:
+                        print("skipping dataset %s, already processed" % parameters['datasetId'])
+                        return False
+                if 'content' in m and 'lemnatec_measurement_metadata' in m['content']:
+                    found_md = True
+
+    if found_ir and found_md:
         return True
     else:
         return False
@@ -116,22 +112,24 @@ def process_dataset(parameters):
     print("...png: %s" % (png_path))
     print("...tif: %s" % (tiff_path))
 
-    print("getting information from json file")
-    center_position, scan_time, fov = getFlir.parse_metadata(metadata) 
-    gps_bounds = getFlir.get_bounding_box(center_position, fov) # get bounding box using gantry position and fov of camera
-
     print("Creating png image")
     raw_data = getFlir.load_flir_data(bin_file) # get raw data from bin file
     im_color = getFlir.create_png(raw_data, png_path) # create png
     print("Uploading output PNGs to dataset")
     extractors.upload_file_to_dataset(png_path, parameters)
+
+    print("getting information from json file for geoTIFF")
+    center_position, scan_time, fov = getFlir.parse_metadata(metadata)
+    if center_position is None or scan_time is None or fov is None:
+        print("error getting metadata; skipping geoTIFF")
+    else:
+        gps_bounds = getFlir.get_bounding_box(center_position, fov) # get bounding box using gantry position and fov of camera
 
-    print("Creating geoTIFF images")
-    tc = getFlir.rawData_to_temperature(raw_data, scan_time, metadata) # get temperature
-    getFlir.create_geotiff_with_temperature(im_color, tc, gps_bounds, tiff_path) # create geotiff
-    print("Uploading output geoTIFFs to dataset")
-    extractors.upload_file_to_dataset(tiff_path, parameters)
-
+        print("Creating geoTIFF images")
+        tc = getFlir.rawData_to_temperature(raw_data, scan_time, metadata) # get temperature
+        getFlir.create_geotiff_with_temperature(im_color, tc, gps_bounds, tiff_path) # create geotiff
+        print("Uploading output geoTIFFs to dataset")
+        extractors.upload_file_to_dataset(tiff_path, parameters)
 
     # Tell Clowder this is completed so subsequent file updates don't daisy-chain
     metadata = {