From 4c175c0aef9c8f6106946b6f32d7f1765e0e57d4 Mon Sep 17 00:00:00 2001 From: Jordan Braiuka Date: Fri, 15 Jan 2016 15:53:28 +1100 Subject: [PATCH] Ran pep8 Shouldn't print empty lines on the individual experiments anymore --- TimestreamAudit.py | 5 +- exif2timestream.py | 265 ++++++++------- test/test_exif2timestream.py | 619 +++++++++++++++++------------------ 3 files changed, 455 insertions(+), 434 deletions(-) diff --git a/TimestreamAudit.py b/TimestreamAudit.py index 50fce26..aa65cea 100644 --- a/TimestreamAudit.py +++ b/TimestreamAudit.py @@ -351,7 +351,10 @@ def output_by_experiment(ordered_dict, input_directory): row.append(perc) else: row.append("") - output.append(row) + for item in row[1:]: + if item != "": + output.append(row) + break output = sorted(output) for line in output: writer.writerow(line) diff --git a/exif2timestream.py b/exif2timestream.py index 3e11f39..0e5013e 100644 --- a/exif2timestream.py +++ b/exif2timestream.py @@ -3,7 +3,6 @@ # pylint:disable=logging-format-interpolation from __future__ import print_function - # Standard library imports import argparse import csv @@ -46,6 +45,7 @@ DATE_NOW_CONSTANTS = {"now", "current"} ongoing = False + def cli_options(): """Return CLI arguments with argparse.""" parser = argparse.ArgumentParser() @@ -58,9 +58,9 @@ def cli_options(): parser.add_argument('-l', '--logdir', help='Directory to contain log files.') parser.add_argument('-c', '--config', help='Path to CSV camera ' - 'config file for normal operation.') + 'config file for normal operation.') parser.add_argument('-g', '--generate', help='Generate a template' - ' camera configuration file at given path.') + ' camera configuration file at given path.') return parser.parse_args() @@ -75,11 +75,11 @@ def date(x): except: raise ValueError + def date_end(x): global ongoing """Converter / validator for date field.""" if isinstance(x, struct_time): - ongoing = False return x if x.lower() in DATE_NOW_CONSTANTS: @@ -92,13 +92,14 @@ def date_end(x): except: raise ValueError + def bool_str(x): """Converts a string to a boolean, even yes/no/true/false.""" if isinstance(x, bool): return x elif isinstance(x, int): return bool(x) - elif (len(x) ==0): + elif (len(x) == 0): return False x = x.strip().lower() if x in {"t", "true", "y", "yes", "f", "false", "n", "no"}: @@ -136,16 +137,18 @@ def resolution_str(x): res_list.append((int(res), None)) return res_list + def dataset(x): "If it exists, appends all neccesary items" if (x): - if len(str(x)) ==1: + if len(str(x)) == 1: return '-F0' + str(x) else: return '-F' + str(x) else: return '' + def cam_pad_str(x): """Pads a numeric string to two digits.""" if len(str(x)) == 1: @@ -219,7 +222,7 @@ class CameraFields(object): ('large_json', 'LARGE_JSON', bool_str), ('json_updates', 'JSON_UPDATES', str), ('sub_folder', 'SUBFOLDER', bool_str) - ) + ) TS_CSV = dict((a, b) for a, b, c in ts_csv_fields) CSV_TS = {v: k for k, v in TS_CSV.items()} @@ -240,9 +243,9 @@ def __init__(self, csv_config_dict): # Ensure required properties are included, and no unknown attributes if not all(key in csv_config_dict for key in self.REQUIRED): raise ValueError('CSV config dict lacks required key/s.') -# TODO: re-enable correctly, to catch illegal keys -# if any(key not in self.TS_CSV for key in csv_config_dict): -# raise ValueError('CSV config dict has unknown key/s.') + # TODO: re-enable correctly, to catch illegal keys + # if any(key not in self.TS_CSV for key in csv_config_dict): + # raise ValueError('CSV config dict has unknown key/s.') # Converts dict keys and calls validation function on each value csv_config_dict = {k: self.SCHEMA[k](v) for k, v in csv_config_dict.items()} @@ -254,6 +257,7 @@ def __init__(self, csv_config_dict): def local(p): """Ensure that pathnames are correct for this system.""" return p.replace(r'\\', '/').replace('/', os.path.sep) + self.source = local(self.source) self.archive_dest = local(self.archive_dest) self.destination = local(self.destination) @@ -276,8 +280,9 @@ def d2s(date): else: return date + def resolution_calc(camera, image): - x=0 + x = 0 try: camera.resolutions[0] = Image.open(image).size except IOError: @@ -291,9 +296,9 @@ def resolution_calc(camera, image): temp = height height = width width = temp - camera.resolutions[0]= (width, height) + camera.resolutions[0] = (width, height) except KeyError: - camera.resolutions[0]=(0,0) + camera.resolutions[0] = (0, 0) if (camera.orientation in ("90", "270")): camera.resolutions[0] = (camera.resolutions[0][1], camera.resolutions[0][0]) for resize_resolution in camera.resolutions: @@ -304,42 +309,45 @@ def resolution_calc(camera, image): new_res = (img[0] * resize_resolution[0] / img[1], resize_resolution[0]) else: new_res = (resize_resolution[0], - img[1] * resize_resolution[0] / img[0]) + img[1] * resize_resolution[0] / img[0]) log.debug("One resolution arguments, '{0:d}'".format(new_res[0])) camera.resolutions[x] = new_res except Exception as e: log.debug("Wouldn't calculate resolution arguments" + str(e)) - x=x+1 + x = x + 1 return camera -def create_small_json(res, camera, full_res, image_resolution, p_start, p_end, ts_end_text, ext, webrootaddr, thumb_image, step): + +def create_small_json(res, camera, full_res, image_resolution, p_start, p_end, ts_end_text, ext, webrootaddr, + thumb_image, step): if (res == "fullres") and (step not in ["cor", "seg"]): folder = "originals" else: folder = "outputs" - if (ext in RAW_FORMATS): step = "raw" else: step = step - if (len(camera.resolutions)>1 and step == 'orig'): + if (len(camera.resolutions) > 1 and step == 'orig'): lower_resolution = True low_res = image_resolution[camera.orientation in ("90", "270")] else: lower_resolution = False low_res = res if not os.path.exists(os.path.join(camera.destination, camera.ts_structure.format(folder=folder, - res=res, step = step))): + res=res, step=step))): os.makedirs(os.path.join(camera.destination, camera.ts_structure.format(folder=folder, - res=res, step = step))) + res=res, step=step))) if os.path.isfile(os.path.join(camera.destination, camera.ts_structure.format(folder=folder, - res=res, step = step), camera.userfriendlyname + '-ts-info.json')): - old_json= open(os.path.join(camera.destination, camera.ts_structure.format(folder=folder, - res=res, step = step), camera.userfriendlyname + '-ts-info.json'), 'r') + res=res, step=step), + camera.userfriendlyname + '-ts-info.json')): + old_json = open(os.path.join(camera.destination, camera.ts_structure.format(folder=folder, + res=res, step=step), + camera.userfriendlyname + '-ts-info.json'), 'r') jdump = eval(old_json.read().replace("null", "None")) old_json.close() - if jdump['posix_start']> mktime(p_start): + if jdump['posix_start'] > mktime(p_start): jdump['posix_start'] = mktime(p_start) jdump['ts_start'] = strftime(TS_DATE_FMT, p_start) if jdump['posix_end'] < mktime(p_end): @@ -350,55 +358,63 @@ def create_small_json(res, camera, full_res, image_resolution, p_start, p_end, t jdump['name'] = camera.userfriendlyname if (camera.method == 'rotate'): jdump['height'] = image_resolution[1] - jdump['height_hires']= full_res[1] - jdump['width']= image_resolution[0] - jdump['width_hires']= full_res[0] + jdump['height_hires'] = full_res[1] + jdump['width'] = image_resolution[0] + jdump['width_hires'] = full_res[0] if (camera.method == 'resize'): jdump['height'] = image_resolution[1] - jdump['height_hires']= full_res[1] - jdump['width']= image_resolution[0] - jdump['width_hires']= full_res[0] - jdump['webroot'] = webrootaddr.format(folder=("outputs" if lower_resolution else "originals"), res=low_res, step =("orig" if ext != 'raw' else "raw")) + jdump['height_hires'] = full_res[1] + jdump['width'] = image_resolution[0] + jdump['width_hires'] = full_res[0] + jdump['webroot'] = webrootaddr.format(folder=("outputs" if lower_resolution else "originals"), res=low_res, + step=("orig" if ext != 'raw' else "raw")) else: jdump = { - 'access':0, + 'access': 0, 'expt': camera.expt, 'owner': camera.project_owner, 'height': image_resolution[1], 'height_hires': full_res[1], 'image_type': ext.upper(), - 'ts_name': camera.fn_structure.format(folder = folder, res = res, step = step).replace(os.path.sep,""), - 'ts_id': '{}-{}-C{}'.format(camera.expt, camera.location, camera.cam_num) + str(camera.datasetID) + ('' if step not in ['cor', 'seg'] else ('-' + step)), - 'name':camera.userfriendlyname , + 'ts_name': camera.fn_structure.format(folder=folder, res=res, step=step).replace(os.path.sep, ""), + 'ts_id': '{}-{}-C{}'.format(camera.expt, camera.location, camera.cam_num) + str(camera.datasetID) + ( + '' if step not in ['cor', 'seg'] else ('-' + step)), + 'name': camera.userfriendlyname, 'period_in_minutes': camera.interval, 'posix_end': mktime(p_end), 'posix_start': mktime(p_start), 'timezone': camera.timezone[0], - 'thumbnails':thumb_image, + 'thumbnails': thumb_image, 'ts_end': ts_end_text, - 'ts_version' :'1', + 'ts_version': '1', 'ts_start': strftime(TS_DATE_FMT, p_start), 'width': image_resolution[0], 'width_hires': full_res[0], - 'webroot':webrootaddr.format(folder=("outputs" if lower_resolution or step in ['cor', 'seg'] else "originals"), res=low_res, step =step), - 'webroot_hires':(webrootaddr.format(folder="outputs" if step in ['cor', 'seg'] else "originals", res="fullres", step=step if step != 'raw' else 'orig')), - 'utc' : 'false', - } + 'webroot': webrootaddr.format( + folder=("outputs" if lower_resolution or step in ['cor', 'seg'] else "originals"), res=low_res, + step=step), + 'webroot_hires': ( + webrootaddr.format(folder="outputs" if step in ['cor', 'seg'] else "originals", res="fullres", + step=step if step != 'raw' else 'orig')), + 'utc': 'false', + } small_json = open(os.path.join(camera.destination, camera.ts_structure.format(folder=folder, - res=res, step = step), camera.userfriendlyname + '-ts-info.json'), 'wb+') + res=res, step=step), + camera.userfriendlyname + '-ts-info.json'), 'wb+') json.dump(jdump, small_json) small_json.close() + def parse_structures(camera): - if len(camera.userfriendlyname) <1: - camera.userfriendlyname = '{}-{}-C{}{}'.format(camera.expt, camera.location, camera.cam_num,camera.datasetID) + if len(camera.userfriendlyname) < 1: + camera.userfriendlyname = '{}-{}-C{}{}'.format(camera.expt, camera.location, camera.cam_num, camera.datasetID) else: - for key, value in camera.__dict__.items(): + for key, value in camera.__dict__.items(): camera.userfriendlyname = camera.userfriendlyname.replace(key.upper(), - str(value)) - camera.userfriendlyname = camera.userfriendlyname.replace(os.path.sep, '') + str(value)) + camera.userfriendlyname = camera.userfriendlyname.replace(os.path.sep, '') """Parse the file structure of the camera for conversion to timestream format.""" @@ -408,12 +424,12 @@ def parse_structures(camera): camera.ts_structure = os.path.join( camera.expt, (camera.location + '-C' + - camera.cam_num + camera.datasetID), + camera.cam_num + camera.datasetID), '{folder}', (camera.expt + '-' + - camera.location + "-C" + - camera.cam_num + - camera.datasetID + "~{res}-{step}" )).replace("_","-") + camera.location + "-C" + + camera.cam_num + + camera.datasetID + "~{res}-{step}")).replace("_", "-") else: # Replace the ts_structure with all the other stuff @@ -429,19 +445,19 @@ def parse_structures(camera): camera.ts_structure = os.path.join( direc, (fname + "~" + "{res}" + "-{step}") - ) + ) if not len(camera.fn_structure) and not camera.fn_structure: camera.fn_structure = camera.expt.replace("_", "-") + \ - '-' + camera.location.replace("_", "-") + \ - '-C' + camera.cam_num.replace("_", "-") +\ - camera.datasetID + \ - '~{res}-{step}' + '-' + camera.location.replace("_", "-") + \ + '-C' + camera.cam_num.replace("_", "-") + \ + camera.datasetID + \ + '~{res}-{step}' else: for key, value in camera.__dict__.items(): camera.fn_structure = camera.fn_structure.replace(key.upper(), str(value)) - camera.fn_structure = camera.fn_structure.replace(os.path.sep, "")\ - .replace("_", "-") + '~{res}-{step}' + camera.fn_structure = camera.fn_structure.replace(os.path.sep, "") \ + .replace("_", "-") + '~{res}-{step}' return camera @@ -528,7 +544,7 @@ def write_exif_date(filename, date_time): return False -def get_file_date(filename, timeshift, round_secs=1, date_mask = DATE_MASK): +def get_file_date(filename, timeshift, round_secs=1, date_mask=DATE_MASK): """Gets a time.struct_time from an image's EXIF, or None if not possible. """ date = None @@ -537,7 +553,7 @@ def get_file_date(filename, timeshift, round_secs=1, date_mask = DATE_MASK): str_date = exif_tags.exif.primary.ExtendedEXIF.DateTimeOriginal date = strptime(str_date, EXIF_DATE_FMT) except (AttributeError, pexif.JpegFile.InvalidFile, struct.error): - # print ("failed pexif") + # print ("failed pexif") pass if not date: with open(filename, "rb") as fh: @@ -547,17 +563,17 @@ def get_file_date(filename, timeshift, round_secs=1, date_mask = DATE_MASK): str_date = exif_tags[EXIF_DATE_TAG].values date = strptime(str_date, EXIF_DATE_FMT) except KeyError: - # print ("failed ExifRead") + # print ("failed ExifRead") pass if not date: - # Try to get datetime from the filename, but not the directory + # Try to get datetime from the filename, but not the directory log.debug("No Exif data in '{}', reading from filename".format( os.path.basename(filename))) # Try and grab the date, we can put a custom mask in here if we want - date = get_time_from_filename(filename,date_mask) + date = get_time_from_filename(filename, date_mask) if date is None: log.debug("Unable to scrape date from '{}'".format(filename)) - # print("Unable to read Exif Data") + # print("Unable to read Exif Data") return None else: if not write_exif_date(filename, date): @@ -659,7 +675,7 @@ def timestreamise_image(image, camera, subsec=0, step="orig"): if camera.orientation and camera.orientation is not 0 and step != "raw": img_array = rotate_image(camera.orientation, dest) write_exif_date(dest, image_date); - elif (len(camera.resolutions)>1) and step != "raw": + elif (len(camera.resolutions) > 1) and step != "raw": img_array = Image.open(dest) if len(camera.resolutions) > 1 and step != "raw": log.info("Going to resize image '{}'".format(dest)) @@ -680,18 +696,20 @@ def timestreamise_image(image, camera, subsec=0, step="orig"): log.debug("Resize failed for unknown reason") raise SkipImage + def rotate_image(rotation, dest): try: img = Image.open(dest) - img = img.rotate(float(rotation), expand =1) + img = img.rotate(float(rotation), expand=1) img.save(dest) return img except IOError: log.debug("Can't Rotate Non JPEG Images {}".format(dest)) + def _dont_clobber(fn, mode="append"): """Ensure we don't overwrite things, using a variety of methods""" - #TODO: will clobber something different if the new filename exists + # TODO: will clobber something different if the new filename exists if os.path.exists(fn): # Deal with SkipImage or StopIteration exceptions, even uninstantiated if isinstance(mode, StopIteration): @@ -714,7 +732,7 @@ def process_image(args): """Do move and copy operations for a camera config and list of images.""" log.debug("Starting to process image") retry = 2 - while(retry): + while (retry): try: image, camera, ext, step = args image_date = get_file_date(image, camera.timeshift, camera.interval * 60) @@ -727,8 +745,8 @@ def process_image(args): return if camera.method == "json": return - if "last_image" in image.lower() : - log.debug ("Skipping file {}, assumed last image".format(image)) + if "last_image" in image.lower(): + log.debug("Skipping file {}, assumed last image".format(image)) return if camera.method == "resize" and (ext not in RAW_FORMATS): img_array = Image.open(image) @@ -745,9 +763,10 @@ def process_image(args): camera.archive_dest, camera.expt, (camera.expt + '-' + - camera.location + "-C" + - camera.cam_num + - camera.datasetID + "~fullres-" + (step if step in (RAW_FORMATS | {"cor", "seg"}) else "orig")).replace("_","-"), + camera.location + "-C" + + camera.cam_num + + camera.datasetID + "~fullres-" + ( + step if step in (RAW_FORMATS | {"cor", "seg"}) else "orig")).replace("_", "-"), os.path.relpath(image, camera.source)) try: os.makedirs(os.path.dirname(archive_image)) @@ -777,8 +796,8 @@ def process_image(args): log.debug("Deleted {}".format(image)) retry = 0 except (struct.error, IOError) as e: - retry =retry - 1 - if (retry >0): + retry = retry - 1 + if (retry > 0): log.debug("Error on image {}, trying again") sleep(1) else: @@ -787,7 +806,6 @@ def process_image(args): def parse_camera_config_csv(filename): - """Parse a camera configuration, yielding localised and validated camera configuration objects.""" if filename is None: @@ -800,7 +818,7 @@ def parse_camera_config_csv(filename): if camera.use: yield parse_structures(camera) except (SkipImage, ValueError) as e: - print ("Error on csv entry", e) + print("Error on csv entry", e) continue @@ -834,10 +852,10 @@ def find_image_files(camera): for fle in files: this_ext = os.path.splitext(fle)[-1].lower().strip(".") if (ext in (this_ext) or (ext == "raw" and this_ext in RAW_FORMATS)) or \ - (ext in ["cor", "seg"] and this_ext == 'jpg'): + (ext in ["cor", "seg"] and this_ext == 'jpg'): fle_path = os.path.join(cur_dir, fle) if camera.fn_parse in fle_path and "last_image" not in fle_path: - count_images+=1 + count_images += 1 print("Found {:5d} Images".format(count_images), end='\r') try: ext_files[ext].append(fle_path) @@ -852,7 +870,7 @@ def find_image_files(camera): if ext in (this_ext) or (ext == "raw" and this_ext in RAW_FORMATS): fle_path = os.path.join(src, fle) if camera.fn_parse in fle_path and "last_image" not in fle_path: - count_images+=1 + count_images += 1 print("Found {:5d} Images".format(count_images), end='\r') try: ext_files[ext].append(fle_path) @@ -862,7 +880,6 @@ def find_image_files(camera): log.info("Found {0} {1} files for camera.".format( len(ext_files), ext)) - return ext_files @@ -910,13 +927,13 @@ def get_resolution(image, camera): width = temp image_resolution = (width, height) except KeyError: - image_resolution=(0,0) + image_resolution = (0, 0) else: try: image_resolution = Image.open(image).size except ValueError: print("Value Error?") - image_resolution=(0,0) + image_resolution = (0, 0) except IOError: image_resolution = (0, 0) folder, res = "originals", 'fullres' @@ -930,7 +947,7 @@ def get_thumbnail_paths(camera, images, res, image_resolution, folder, step='ori url = "http://phenocam.anu.edu.au/cloud/a_data" webrootaddr = "http://phenocam.anu.edu.au/cloud/a_data{}/{}".format( camera.destination.split("a_data")[-1], - camera.ts_structure if camera.ts_structure else camera.location).replace("\\","/") + camera.ts_structure if camera.ts_structure else camera.location).replace("\\", "/") thumb_image = [] if len(images) > 0: sep = '/' @@ -939,7 +956,7 @@ def get_thumbnail_paths(camera, images, res, image_resolution, folder, step='ori start = 0 else: max = 3 - start = (len(images)//2) -1 + start = (len(images) // 2) - 1 for i in range(max): try: image_date = get_file_date(images[start + i], camera.timeshift, @@ -948,21 +965,23 @@ def get_thumbnail_paths(camera, images, res, image_resolution, folder, step='ori image_date, make_timestream_name(camera, res, step)) thumb_image.append(sep.join([ camera.destination, os.path.dirname(camera.ts_structure).format(folder=folder), - os.path.basename(camera.ts_structure).format(res=res, step=step), ts_image]).replace("\\","/")) + os.path.basename(camera.ts_structure).format(res=res, step=step), ts_image]).replace("\\", "/")) except (SkipImage): pass for i in range(len(thumb_image)): if thumb_image[i]: thumb_image[i] = url + thumb_image[i].split("a_data")[-1] - if len(camera.resolutions)>1: - thumb_image[i] = thumb_image[i].format(folder="outputs", res = camera.resolutions[1][camera.orientation in ("90", "270")]) + if len(camera.resolutions) > 1: + thumb_image[i] = thumb_image[i].format(folder="outputs", + res=camera.resolutions[1][camera.orientation in ("90", "270")]) else: - thumb_image[i] = thumb_image[i].format(folder="originals", res = "orig") + thumb_image[i] = thumb_image[i].format(folder="originals", res="orig") return webrootaddr, thumb_image + def get_actual_start_end(camera, images, ext): earlier = True - j=0 + j = 0 my_ext_images = []; date = '' for image in images: @@ -971,34 +990,36 @@ def get_actual_start_end(camera, images, ext): my_ext_images.append(image); elif (my_ext in RAW_FORMATS) and (ext == "raw"): my_ext_images.append(image); - while earlier and (j<= len(my_ext_images)-1): + while earlier and (j <= len(my_ext_images) - 1): date = get_file_date(my_ext_images[j], camera.timeshift, camera.interval * 60) if (date >= camera.expt_start) and (date is not None): earlier = False - j+=1 + j += 1 if not (date): date = camera.expt_start p_start = date later = True - j = len(my_ext_images)-1 + j = len(my_ext_images) - 1 date = None - while later and j>=0: + while later and j >= 0: date = get_file_date(my_ext_images[j], camera.timeshift, camera.interval * 60) if (date <= camera.expt_end) and (date is not None): later = False - j-=1 + j -= 1 if date is None: date = camera.expt_end p_end = date return p_start, p_end + def find_empty_dirs(root_dir): for dirpath, dirs, files in os.walk(root_dir, topdown=False): - if(len(files) is 1 and "thumbs.db" in files): - os.remove(os.path.join(dirpath,"thumbs.db")) - if (not dirs and not files) or len(os.listdir(dirpath))==0: + if (len(files) is 1 and "thumbs.db" in files): + os.remove(os.path.join(dirpath, "thumbs.db")) + if (not dirs and not files) or len(os.listdir(dirpath)) == 0: os.rmdir(dirpath) + def process_camera(camera, ext, images, n_threads=1): """Process a set of images for one extension for a single camera.""" if ext in ["cor", "seg"]: @@ -1014,19 +1035,20 @@ def process_camera(camera, ext, images, n_threads=1): images = new_images p_start, p_end = get_actual_start_end(camera, images, ext) try: - my_image = (x for x in images if ((os.path.splitext(x)[-1].lower().strip(".") == ext) or (os.path.splitext(x)[-1].lower().strip(".") in RAW_FORMATS and ext == "raw"))).next() + my_image = (x for x in images if ((os.path.splitext(x)[-1].lower().strip(".") == ext) or ( + os.path.splitext(x)[-1].lower().strip(".") in RAW_FORMATS and ext == "raw"))).next() except StopIteration: - return + return camera = resolution_calc(camera, my_image) res, image_resolution, folder = get_resolution(my_image, camera) - if (len(camera.resolutions) >1): + if (len(camera.resolutions) > 1): low_res = camera.resolutions[1][camera.orientation in ("90", "270")] low_folder = "outputs" else: low_res = "fullres" low_folder = "originals" webrootaddr, thumb_image = get_thumbnail_paths(camera, images, low_res, image_resolution, low_folder, step=step) - webrootaddr = webrootaddr.replace("\\","/") + webrootaddr = webrootaddr.replace("\\", "/") # TODO: sort out the whole subsecond clusterfuck if n_threads == 1: @@ -1054,7 +1076,7 @@ def process_camera(camera, ext, images, n_threads=1): fullres = (image_resolution[1], image_resolution[0]) else: fullres = image_resolution - if len(camera.resolutions) >1 and ext not in RAW_FORMATS: + if len(camera.resolutions) > 1 and ext not in RAW_FORMATS: new_res = camera.resolutions[1] else: new_res = fullres @@ -1070,43 +1092,45 @@ def process_camera(camera, ext, images, n_threads=1): 'height_hires': fullres[1], 'height': new_res[1], 'image_type': 'JPG' if ext not in RAW_FORMATS else 'RAW', - 'ts_id': '{}-{}-C{}'.format(camera.expt, camera.location, camera.cam_num ) + str(camera.datasetID) + ('' if step not in ['cor', 'seg'] else ('-' + step)), - 'name':camera.userfriendlyname + ('' if step not in ['cor', 'seg'] else ('-' + step)), - 'owner':camera.project_owner, + 'ts_id': '{}-{}-C{}'.format(camera.expt, camera.location, camera.cam_num) + str(camera.datasetID) + ( + '' if step not in ['cor', 'seg'] else ('-' + step)), + 'name': camera.userfriendlyname + ('' if step not in ['cor', 'seg'] else ('-' + step)), + 'owner': camera.project_owner, 'period_in_minutes': camera.interval, 'posix_end': mktime(p_end), 'posix_start': mktime(p_start), 'thumbnails': thumb_image, 'timezone': camera.timezone[0], - 'ts_name' : camera.fn_structure.format(folder = folder, res = res, step = step).replace(os.path.sep,""), + 'ts_name': camera.fn_structure.format(folder=folder, res=res, step=step).replace(os.path.sep, ""), 'ts_end': ts_end_text, 'ts_start': strftime(TS_DATE_FMT, p_start), 'ts_version': '1', 'utc': "false", - 'webroot_hires':(webrootaddr.format(folder="originals" if step in ["orig", "raw"] else "outputs", res="fullres", step=step)), - 'webroot':webrootaddr.format(folder="outputs", res=new_res[camera.orientation in ("90", - "270")], step =step), + 'webroot_hires': ( + webrootaddr.format(folder="originals" if step in ["orig", "raw"] else "outputs", res="fullres", step=step)), + 'webroot': webrootaddr.format(folder="outputs", res=new_res[camera.orientation in ("90", + "270")], step=step), 'width_hires': fullres[0], 'width': new_res[0] - } + } if (camera.json_updates) and ext != "raw" and camera.large_json: for key, value in jdump.items(): if not (key.lower() in camera.json_updates.lower()): if not (key.lower() in ('ts_name')): jdump.pop(key, None) - create_small_json("fullres", camera,fullres, new_res, p_start, p_end, ts_end_text, ext, webrootaddr, thumb_image, step) + create_small_json("fullres", camera, fullres, new_res, p_start, p_end, ts_end_text, ext, webrootaddr, thumb_image, + step) if ext not in RAW_FORMATS: for resize_res in camera.resolutions[1:]: new_res = resize_res - create_small_json(new_res[camera.orientation in ("90", "270")], camera, fullres, new_res, p_start, p_end, ts_end_text, ext, webrootaddr, thumb_image, step) + create_small_json(new_res[camera.orientation in ("90", "270")], camera, fullres, new_res, p_start, p_end, + ts_end_text, ext, webrootaddr, thumb_image, step) if ext != 'raw' and camera.large_json: return {k: v for k, v in jdump.items()} else: return False - - def main(configfile, n_threads=1, logdir=None, debug=False): """The main loop of the module, do the renaming in parallel etc.""" setup_logs(logdir, debug) @@ -1136,22 +1160,22 @@ def main(configfile, n_threads=1, logdir=None, debug=False): len(images), ext)) n_images += len(images) j_dump = process_camera(camera, ext, sorted(images), - n_threads) + n_threads) # if (camera.large_json): if (j_dump): json_dump.append(j_dump) - jpath = os.path.join(camera.destination) #, os.path.dirname( - # camera.ts_structure.format(folder='', res='', cam='')) + jpath = os.path.join(camera.destination) # , os.path.dirname( + # camera.ts_structure.format(folder='', res='', cam='')) try: os.makedirs(jpath) except OSError: if not os.path.exists(jpath): log.warn("Could not make dir '{}', skipping images" .format(jpath)) - if (len(json_dump)>0): + if (len(json_dump) > 0): with open(os.path.join(jpath, 'all_cameras.json'), 'w') as fname: json.dump(json_dump, fname) - #remove any empty directories in source + # remove any empty directories in source if camera.method == "archive": empty = find_empty_dirs(camera.source) secs_taken = time() - start_time @@ -1172,6 +1196,7 @@ def gen_config(fname): opts = cli_options() if opts.version: from ._version import get_versions + print("Version {}".format(get_versions()['version'])) sys.exit(0) if opts.generate: diff --git a/test/test_exif2timestream.py b/test/test_exif2timestream.py index 740e429..0fea17d 100644 --- a/test/test_exif2timestream.py +++ b/test/test_exif2timestream.py @@ -19,7 +19,7 @@ from .. import DeleteImagesByTime as dbt import csv -PIL= True +PIL = True try: from PIL import Image except ImportError: @@ -58,17 +58,18 @@ class TestExifTraitcapture(unittest.TestCase): 'CAMERA_TIMEZONE': '1100', 'USE': '1', 'USER': 'Glasshouses', - 'TS_STRUCTURE': os.path.join('BVZ00000',"EUC-R01C01-C01-F01", "{folder}", 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}'), + 'TS_STRUCTURE': os.path.join('BVZ00000', "EUC-R01C01-C01-F01", "{folder}", + 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}'), 'FN_PARSE': '', 'PROJECT_OWNER': '', 'FILENAME_DATE_MASK': '', 'FN_STRUCTURE': 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}', 'ORIENTATION': '', - 'DATASETID':1, - 'TIMESHIFT':0, - 'USERFRIENDLYNAME':'BVZ00000-EUC-R01C01-C01-F01', - 'JSON_UPDATES':'', - 'LARGE_JSON':0, + 'DATASETID': 1, + 'TIMESHIFT': 0, + 'USERFRIENDLYNAME': 'BVZ00000-EUC-R01C01-C01-F01', + 'JSON_UPDATES': '', + 'LARGE_JSON': 0, 'SUBFOLDER': 1 } config_list_delete = { @@ -85,115 +86,116 @@ class TestExifTraitcapture(unittest.TestCase): } original_jpg_json = { - "access":0, - "width":1920, - "period_in_minutes":5, - "expt":"BVZ00000", - "ts_end":"now", - "height":1280, - "posix_start":1384289700.0, - "image_type":"JPG", - "height_hires":3456, - "timezone":11, - "width_hires":5184, - "webroot_hires":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", - "ts_name":"BVZ00000-EUC-R01C01-C01-F01~fullres-orig", - "ts_start":'2013_11_12_20_55_00', - 'thumbnails': [ - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_12_20_55_00_00.jpg', - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_11/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_11_40_00_00.jpg', - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_12/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_12_25_00_00.jpg' - ], - "owner":"", - "webroot":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig", - "name":"BVZ00000-EUC-R01C01-C01-F01", - "ts_id":"BVZ00000-EUC-R01C01-C01-F01", - "posix_end":1385209500.0, - 'ts_version': '1', - 'utc' : 'false', + "access": 0, + "width": 1920, + "period_in_minutes": 5, + "expt": "BVZ00000", + "ts_end": "now", + "height": 1280, + "posix_start": 1384289700.0, + "image_type": "JPG", + "height_hires": 3456, + "timezone": 11, + "width_hires": 5184, + "webroot_hires": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", + "ts_name": "BVZ00000-EUC-R01C01-C01-F01~fullres-orig", + "ts_start": '2013_11_12_20_55_00', + 'thumbnails': [ + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_12_20_55_00_00.jpg', + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_11/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_11_40_00_00.jpg', + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_12/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_12_25_00_00.jpg' + ], + "owner": "", + "webroot": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig", + "name": "BVZ00000-EUC-R01C01-C01-F01", + "ts_id": "BVZ00000-EUC-R01C01-C01-F01", + "posix_end": 1385209500.0, + 'ts_version': '1', + 'utc': 'false', } raw_json = { - "access":0, - "width":3456, - "period_in_minutes":5, - "expt":"BVZ00000", - "ts_end":"now", - "height":5184, - "posix_start":1384289700.0, - "image_type":"RAW", - "height_hires":5184, - "timezone":11, - "width_hires":3456, - "webroot_hires":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", - "ts_name":"BVZ00000-EUC-R01C01-C01-F01~fullres-raw", - "ts_start":'2013_11_12_20_55_00', - 'thumbnails': ['http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_12_20_55_00_00.jpg'], - "owner":"", - "webroot":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-raw", - "name":"BVZ00000-EUC-R01C01-C01-F01", - "ts_id":"BVZ00000-EUC-R01C01-C01-F01", - "posix_end":1384289700.0, - 'ts_version': '1', - 'utc' : 'false', + "access": 0, + "width": 3456, + "period_in_minutes": 5, + "expt": "BVZ00000", + "ts_end": "now", + "height": 5184, + "posix_start": 1384289700.0, + "image_type": "RAW", + "height_hires": 5184, + "timezone": 11, + "width_hires": 3456, + "webroot_hires": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", + "ts_name": "BVZ00000-EUC-R01C01-C01-F01~fullres-raw", + "ts_start": '2013_11_12_20_55_00', + 'thumbnails': [ + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_12_20_55_00_00.jpg'], + "owner": "", + "webroot": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-raw", + "name": "BVZ00000-EUC-R01C01-C01-F01", + "ts_id": "BVZ00000-EUC-R01C01-C01-F01", + "posix_end": 1384289700.0, + 'ts_version': '1', + 'utc': 'false', } original_single_jpg_json = { - "access":0, - "width":5184, - "period_in_minutes":5, - "expt":"BVZ00000", - "ts_end":"now", - "height":3456, - "posix_start":1384289700.0, - "image_type":"JPG", - "height_hires":3456, - "timezone":11, - "width_hires":5184, - "webroot_hires":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", - "ts_name":"BVZ00000-EUC-R01C01-C01-F01~fullres-orig", - "ts_start":'2013_11_12_20_55_00', - 'thumbnails': [ - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~fullres-orig_2013_11_12_20_55_00_00.jpg', - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig/2013/2013_11/2013_11_23/2013_11_23_11/BVZ00000-EUC-R01C01-C01-F01~fullres-orig_2013_11_23_11_40_00_00.jpg', - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig/2013/2013_11/2013_11_23/2013_11_23_12/BVZ00000-EUC-R01C01-C01-F01~fullres-orig_2013_11_23_12_25_00_00.jpg' - ], - "owner":"", - "webroot":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", - "name":"BVZ00000-EUC-R01C01-C01-F01", - "ts_id":"BVZ00000-EUC-R01C01-C01-F01", - 'ts_version': '1', - "posix_end":1385209500.0, - 'utc' : 'false', + "access": 0, + "width": 5184, + "period_in_minutes": 5, + "expt": "BVZ00000", + "ts_end": "now", + "height": 3456, + "posix_start": 1384289700.0, + "image_type": "JPG", + "height_hires": 3456, + "timezone": 11, + "width_hires": 5184, + "webroot_hires": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", + "ts_name": "BVZ00000-EUC-R01C01-C01-F01~fullres-orig", + "ts_start": '2013_11_12_20_55_00', + 'thumbnails': [ + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~fullres-orig_2013_11_12_20_55_00_00.jpg', + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig/2013/2013_11/2013_11_23/2013_11_23_11/BVZ00000-EUC-R01C01-C01-F01~fullres-orig_2013_11_23_11_40_00_00.jpg', + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig/2013/2013_11/2013_11_23/2013_11_23_12/BVZ00000-EUC-R01C01-C01-F01~fullres-orig_2013_11_23_12_25_00_00.jpg' + ], + "owner": "", + "webroot": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", + "name": "BVZ00000-EUC-R01C01-C01-F01", + "ts_id": "BVZ00000-EUC-R01C01-C01-F01", + 'ts_version': '1', + "posix_end": 1385209500.0, + 'utc': 'false', } resized_jpg_json = { - "access":0, - "width":1920, - "period_in_minutes":5, - "expt":"BVZ00000", - "ts_end":"now", - "height":1280, - "posix_start":1384289700.0, - "image_type":"JPG", - "height_hires":3456, - "timezone":11, - "width_hires":5184, - "webroot_hires":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", - "ts_name":"BVZ00000-EUC-R01C01-C01-F01~1920-orig", - "ts_start":'2013_11_12_20_55_00', - 'thumbnails': [ - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_12_20_55_00_00.jpg', - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_11/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_11_40_00_00.jpg', - 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_12/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_12_25_00_00.jpg' - ], - "owner":"", - "webroot":"http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig", - "name":"BVZ00000-EUC-R01C01-C01-F01", - "ts_id":"BVZ00000-EUC-R01C01-C01-F01", - 'ts_version': '1', - "posix_end":1385209500.0, - 'utc' : 'false', - } + "access": 0, + "width": 1920, + "period_in_minutes": 5, + "expt": "BVZ00000", + "ts_end": "now", + "height": 1280, + "posix_start": 1384289700.0, + "image_type": "JPG", + "height_hires": 3456, + "timezone": 11, + "width_hires": 5184, + "webroot_hires": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/originals/BVZ00000-EUC-R01C01-C01-F01~fullres-orig", + "ts_name": "BVZ00000-EUC-R01C01-C01-F01~1920-orig", + "ts_start": '2013_11_12_20_55_00', + 'thumbnails': [ + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_12/2013_11_12_20/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_12_20_55_00_00.jpg', + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_11/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_11_40_00_00.jpg', + 'http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig/2013/2013_11/2013_11_23/2013_11_23_12/BVZ00000-EUC-R01C01-C01-F01~1920-orig_2013_11_23_12_25_00_00.jpg' + ], + "owner": "", + "webroot": "http://phenocam.anu.edu.au/cloud/a_data./test/out/timestreams/BVZ00000/EUC-R01C01-C01-F01/outputs/BVZ00000-EUC-R01C01-C01-F01~1920-orig", + "name": "BVZ00000-EUC-R01C01-C01-F01", + "ts_id": "BVZ00000-EUC-R01C01-C01-F01", + 'ts_version': '1', + "posix_end": 1385209500.0, + 'utc': 'false', + } r_fullres_path = os.path.join( out_dirname, "timestreams", "BVZ00000", "EUC-R01C01-C01-F01", @@ -209,14 +211,14 @@ class TestExifTraitcapture(unittest.TestCase): ) r_datetime_path = os.path.join( out_dirname, "timestreams", "BVZ00000", "EUC-R01C01-C01-F01", - "originals",'BVZ00000-EUC-R01C01-C01-F01~fullres-orig', '2013', '2013_11', + "originals", 'BVZ00000-EUC-R01C01-C01-F01~fullres-orig', '2013', '2013_11', '2013_11_04', '2013_11_04_02', 'BVZ00000-EUC-R01C01-C01-F01~fullres-orig_2013_11_04_22_05_00_00.JPG' ) r_raw_path = os.path.join( out_dirname, "timestreams", "BVZ00000", "EUC-R01C01-C01-F01", - "originals",'BVZ00000-EUC-R01C01-C01-F01~fullres-raw', '2013', '2013_11', + "originals", 'BVZ00000-EUC-R01C01-C01-F01~fullres-raw', '2013', '2013_11', '2013_11_12', '2013_11_12_20', 'BVZ00000-EUC-R01C01-C01-F01~fullres-raw_2013_11_12_20_55_00_00' '.CR2' @@ -253,7 +255,6 @@ def setUp(self): shutil.copytree("./test/unburnable", img_dir) self.camera = e2t.CameraFields(self.camera) - def wipe_output(self): cam = self.camera_both self.camera_raw = copy.deepcopy(cam) @@ -269,6 +270,7 @@ def test_main_expt_dates(self): os.remove(self.r_fullres_path) e2t.main(self.test_config_dates_csv, logdir=self.out_dirname) self.assertFalse(path.exists(self.r_fullres_path)) + # # # test for localise_cam_config # def test_localise_cam_config(self): @@ -379,11 +381,11 @@ def test_make_timestream_name_params(self): # tests for find_image_files def test_find_image_files(self): expt = {"jpg": {path.join(self.camupload_dir, x) for x in [ - os.path.join('jpg', 'IMG_0001.JPG'), - os.path.join('jpg', 'IMG_0002.JPG'), - os.path.join('jpg', 'IMG_0630.JPG'), - os.path.join('jpg', 'IMG_0633.JPG'), - os.path.join('jpg', 'whroo2013_11_04_02_02_55M.jpg')] + os.path.join('jpg', 'IMG_0001.JPG'), + os.path.join('jpg', 'IMG_0002.JPG'), + os.path.join('jpg', 'IMG_0630.JPG'), + os.path.join('jpg', 'IMG_0633.JPG'), + os.path.join('jpg', 'whroo2013_11_04_02_02_55M.jpg')] }, "raw": {path.join(self.camupload_dir, os.path.join('raw', 'IMG_0001.CR2'))}, } @@ -434,18 +436,19 @@ def test_parse_camera_config_csv(self): 'sunset': (22, 0), 'use': True, 'user': 'Glasshouses', - 'ts_structure': os.path.sep.join(['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), + 'ts_structure': os.path.sep.join( + ['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), 'project_owner': '', - 'filename_date_mask':'', + 'filename_date_mask': '', 'fn_parse': '', 'fn_structure': 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}', 'orientation': '', - 'timeshift':'', - 'datasetID':'-F01', - 'json_updates':'', - 'large_json':False, - 'sub_folder':True, - 'userfriendlyname':'BVZ00000-EUC-R01C01-C01-F01' + 'timeshift': '', + 'datasetID': '-F01', + 'json_updates': '', + 'large_json': False, + 'sub_folder': True, + 'userfriendlyname': 'BVZ00000-EUC-R01C01-C01-F01' } ] result = e2t.parse_camera_config_csv(self.test_config_csv) @@ -499,21 +502,15 @@ def test_check_resize_img(self): pass self.assertEqual(w, new_width) - - def test_main(self): e2t.main(self.test_config_csv, logdir=self.out_dirname) self.assertTrue(path.exists(self.r_fullres_path)) - - def test_main_raw(self): e2t.main(self.test_config_raw_csv, logdir=self.out_dirname) self.assertTrue(path.exists(self.r_fullres_path)) self.assertTrue(path.exists(self.r_raw_path)) - - def test_main_threads(self): # with a good value for threads e2t.main(self.test_config_csv, logdir=self.out_dirname, n_threads=1) @@ -531,8 +528,6 @@ def test_main_threads_one(self): # deterministic self._md5test(self.r_fullres_path, "76ee6fb2f5122d2f5815101ec66e7cb8") - - def test_orientation(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -540,8 +535,8 @@ def test_orientation(self): orig = Image.open(self.r_fullres_path).size e2t.rotate_image(90, self.r_fullres_path) after = Image.open(self.r_fullres_path).size - self.assertGreater(2, abs(orig[0]- after[1])) - self.assertGreater(2, abs(orig[1]- after[0])) + self.assertGreater(2, abs(orig[0] - after[1])) + self.assertGreater(2, abs(orig[1] - after[0])) e2t.rotate_image(270, self.r_fullres_path) self.assertEqual(True, e2t.write_exif_date(self.r_fullres_path, image_date)) new_image_date = e2t.get_file_date(self.r_fullres_path, 0, 60) @@ -554,37 +549,36 @@ def test_resize_main(self): resize_new['RESOLUTIONS'] = "original~1920" resize_new['METHOD'] = 'move' resize_new['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) - resize_new['TS_STRUCTURE'] = os.path.join('BVZ00000',"EUC-R01C01-C01-F01", "{folder}", 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}') + resize_new['TS_STRUCTURE'] = os.path.join('BVZ00000', "EUC-R01C01-C01-F01", "{folder}", + 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}') rotate = e2t.CameraFields(resize_new) self.wipe_output() - for ext,images in e2t.find_image_files(rotate).items(): + for ext, images in e2t.find_image_files(rotate).items(): images = sorted(images) e2t.process_camera(rotate, ext, images, n_threads=1) new = Image.open(self.r_resize_path).size self.assertEqual(new[0], 1920) self.assertEqual(new[1], 1280) - # Check the JSONS + # Check the JSONS file_path = os.path.join(rotate.destination, rotate.ts_structure.format( folder='originals', res='fullres', step='orig'), rotate.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') original_json = eval(open(file_path).read()) test_json = copy.deepcopy(self.original_jpg_json) self.assertDictEqual(original_json, test_json) - - # Finally the resized json + # Finally the resized json file_path_resized = os.path.join(rotate.destination, rotate.ts_structure.format( folder='outputs', res='1920', step='orig'), rotate.userfriendlyname - + '-ts-info.json') - resized_json= eval(open(file_path_resized).read()) + + '-ts-info.json') + resized_json = eval(open(file_path_resized).read()) - resized_test_json =copy.deepcopy(self.resized_jpg_json) + resized_test_json = copy.deepcopy(self.resized_jpg_json) self.assertDictEqual(resized_json, resized_test_json) - def test_rotate_main(self): rotate_new = copy.deepcopy(self.camera_both) rotate_new['EXPT_END'] = "now" @@ -595,19 +589,19 @@ def test_rotate_main(self): rotate = e2t.CameraFields(rotate_new) orig = Image.open(self.jpg_testfile).size self.wipe_output() - for ext,images in e2t.find_image_files(rotate).items(): + for ext, images in e2t.find_image_files(rotate).items(): images = sorted(images) e2t.process_camera(rotate, ext, images, n_threads=1) new = Image.open(self.r_fullres_path).size self.assertNotEquals(orig, new) - self.assertLess(abs(orig[1]- new[0]), 2) - self.assertLess(abs(orig[0]- new[1]), 2) + self.assertLess(abs(orig[1] - new[0]), 2) + self.assertLess(abs(orig[0] - new[1]), 2) - # Check the JSONS + # Check the JSONS file_path = os.path.join(rotate.destination, rotate.ts_structure.format( folder='originals', res='fullres', step='orig'), rotate.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') original_json = eval(open(file_path).read()) test_json = copy.deepcopy(self.original_single_jpg_json) test_json['height'] = 5184 @@ -616,31 +610,29 @@ def test_rotate_main(self): test_json['width_hires'] = 3456 self.assertDictEqual(original_json, test_json) - - - def test_rotate_resize_main(self): rotate_resize_new = copy.deepcopy(self.camera_both) rotate_resize_new['EXPT_END'] = "now" rotate_resize_new['IMAGE_TYPES'] = "jpg" rotate_resize_new['ORIENTATION'] = 90 rotate_resize_new['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) - rotate_resize_new['TS_STRUCTURE'] = os.path.join('BVZ00000',"EUC-R01C01-C01-F01", "{folder}", 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}') + rotate_resize_new['TS_STRUCTURE'] = os.path.join('BVZ00000', "EUC-R01C01-C01-F01", "{folder}", + 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}') rotate_resize_new['RESOLUTIONS'] = "original~1920" rotate_resize = e2t.CameraFields(rotate_resize_new) self.wipe_output() - for ext,images in e2t.find_image_files(rotate_resize).items(): + for ext, images in e2t.find_image_files(rotate_resize).items(): images = sorted(images) e2t.process_camera(rotate_resize, ext, images, n_threads=1) new = Image.open(self.r_resize_path).size self.assertEqual(new[1], 1920) self.assertEqual(new[0], 1280) - # Check the JSONS + # Check the JSONS file_path = os.path.join(rotate_resize.destination, rotate_resize.ts_structure.format( folder='originals', res='fullres', step='orig'), rotate_resize.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') original_json = eval(open(file_path).read()) test_json = copy.deepcopy(self.original_jpg_json) test_json['height'] = 1920 @@ -651,8 +643,8 @@ def test_rotate_resize_main(self): file_path_resized = os.path.join(rotate_resize.destination, rotate_resize.ts_structure.format( folder='outputs', res='1920', step='orig'), rotate_resize.userfriendlyname - + '-ts-info.json') - resized_json= eval(open(file_path_resized).read()) + + '-ts-info.json') + resized_json = eval(open(file_path_resized).read()) test_json_resized = copy.deepcopy(self.resized_jpg_json) test_json_resized['height'] = 1920 test_json_resized['height_hires'] = 5184 @@ -662,37 +654,38 @@ def test_rotate_resize_main(self): def test_start_end(self): start_end = e2t.CameraFields({ - 'ARCHIVE_DEST': os.path.sep.join(['.', 'test', 'out', 'archive']), - 'CAMERA_TIMEZONE': "11", - 'EXPT': 'BVZ00000', - 'DESTINATION': os.path.sep.join(['.', 'test', 'out', 'timestreams']), - 'CAM_NUM': '01', - 'EXPT_END': "now", - 'EXPT_START': "2002_01_01", - 'INTERVAL': 5, - 'IMAGE_TYPES': "jpg~raw", - 'LOCATION': 'EUC-R01C01', - 'METHOD': 'move', - 'MODE': 'batch', - 'RESOLUTIONS': 'original~1920', - 'SOURCE': os.path.sep.join(['.', 'test', 'img', 'camupload']), - 'SUNRISE': "0500", - 'SUNSET': "2200", - 'USE': True, - 'USER': 'Glasshouses', - 'TS_STRUCTURE': os.path.sep.join(['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), - 'PROJECT_OWNER': '', - 'FILENAME_DATE_MASK':'', - 'FN_PARSE': '', - 'FN_STRUCTURE': 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}', - 'ORIENTATION': '90', - 'TIMESHIFT':'', - 'DATASETID':'-F01', - 'JSON_UPDATES':'', - 'SUBFOLDER':True, - 'LARGE_JSON':0, - 'USERFRIENDLYNAME':'BVZ00000-EUC-R01C01-C01-F01' - }) + 'ARCHIVE_DEST': os.path.sep.join(['.', 'test', 'out', 'archive']), + 'CAMERA_TIMEZONE': "11", + 'EXPT': 'BVZ00000', + 'DESTINATION': os.path.sep.join(['.', 'test', 'out', 'timestreams']), + 'CAM_NUM': '01', + 'EXPT_END': "now", + 'EXPT_START': "2002_01_01", + 'INTERVAL': 5, + 'IMAGE_TYPES': "jpg~raw", + 'LOCATION': 'EUC-R01C01', + 'METHOD': 'move', + 'MODE': 'batch', + 'RESOLUTIONS': 'original~1920', + 'SOURCE': os.path.sep.join(['.', 'test', 'img', 'camupload']), + 'SUNRISE': "0500", + 'SUNSET': "2200", + 'USE': True, + 'USER': 'Glasshouses', + 'TS_STRUCTURE': os.path.sep.join( + ['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), + 'PROJECT_OWNER': '', + 'FILENAME_DATE_MASK': '', + 'FN_PARSE': '', + 'FN_STRUCTURE': 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}', + 'ORIENTATION': '90', + 'TIMESHIFT': '', + 'DATASETID': '-F01', + 'JSON_UPDATES': '', + 'SUBFOLDER': True, + 'LARGE_JSON': 0, + 'USERFRIENDLYNAME': 'BVZ00000-EUC-R01C01-C01-F01' + }) # Check Jpeg images_both = e2t.find_image_files(start_end) images = images_both['jpg'] @@ -718,21 +711,20 @@ def test_start_end(self): self.assertEqual(start, start_end.expt_start) self.assertEqual(end, start_end.expt_end) - def test_timeshift(self): before = e2t.get_file_date(self.r_fullres_path, "", 60) after = e2t.get_file_date(self.r_fullres_path, "10", 60) self.assertNotEqual(before, after) shift_to_equal = datetime.timedelta(hours=10) - before = (datetime.datetime.fromtimestamp(time.mktime(before))+shift_to_equal).timetuple() + before = (datetime.datetime.fromtimestamp(time.mktime(before)) + shift_to_equal).timetuple() self.assertEqual(before, after) def test_filename_parse(self): expt = {"jpg": {path.join(self.camupload_dir, x) for x in [ - os.path.join('jpg', 'IMG_0001.JPG'), - os.path.join('jpg', 'IMG_0002.JPG'), - os.path.join('jpg', 'IMG_0630.JPG'), - os.path.join('jpg', 'IMG_0633.JPG'),] + os.path.join('jpg', 'IMG_0001.JPG'), + os.path.join('jpg', 'IMG_0002.JPG'), + os.path.join('jpg', 'IMG_0630.JPG'), + os.path.join('jpg', 'IMG_0633.JPG'), ] }, } self.camera.fn_parse = "IMG_" @@ -740,7 +732,6 @@ def test_filename_parse(self): self.assertSetEqual(set(got["jpg"]), expt["jpg"]) self.assertSetEqual(set(got["jpg"]), expt["jpg"]) - def test_json_mode(self): no_large_json = copy.deepcopy(self.camera_both) no_large_json['RESOLUTIONS'] = "original~1920" @@ -748,22 +739,22 @@ def test_json_mode(self): no_large_json['EXPT_END'] = "now" no_large_json = e2t.CameraFields(no_large_json) self.wipe_output() - for ext,images in e2t.find_image_files(no_large_json).items(): + for ext, images in e2t.find_image_files(no_large_json).items(): images = sorted(images) output = e2t.process_camera(no_large_json, ext, images, n_threads=1) - self.assertEqual(False,output) - # Check the Mini-Jsons + self.assertEqual(False, output) + # Check the Mini-Jsons # First the original value file_path = os.path.join(no_large_json.destination, no_large_json.ts_structure.format( folder='originals', res='fullres', step='orig'), no_large_json.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') original_json = eval(open(file_path).read()) test_json = copy.deepcopy(self.original_jpg_json) self.assertDictEqual(original_json, test_json) # Then the Raw file_path_raw = os.path.join(no_large_json.destination, no_large_json.ts_structure.format( folder='originals', res='fullres', step='raw'), no_large_json.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') temp = open(file_path_raw) temp = str(temp.read()) raw_original_json = eval(str(temp)) @@ -773,17 +764,17 @@ def test_json_mode(self): # Finally the resized json file_path_resized = os.path.join(no_large_json.destination, no_large_json.ts_structure.format( folder='outputs', res='1920', step='orig'), no_large_json.userfriendlyname - + '-ts-info.json') - resized_json= eval(open(file_path_resized).read()) + + '-ts-info.json') + resized_json = eval(open(file_path_resized).read()) - resized_test_json = copy.deepcopy(self.resized_jpg_json) + resized_test_json = copy.deepcopy(self.resized_jpg_json) self.assertDictEqual(resized_json, resized_test_json) # Delete all the JSONS for file in [file_path, file_path_raw, file_path_resized]: os.remove(file) - json_mode = copy.deepcopy(self.camera_both) + json_mode = copy.deepcopy(self.camera_both) json_mode['RESOLUTIONS'] = "original~1920" json_mode['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) json_mode['SOURCE'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) @@ -792,95 +783,93 @@ def test_json_mode(self): json_mode['LARGE_JSON'] = 'True' json_mode = e2t.CameraFields(json_mode) - - both_image_types =e2t.find_image_files(json_mode) + both_image_types = e2t.find_image_files(json_mode) images = both_image_types["raw"] images = sorted(images) output = e2t.process_camera(json_mode, "raw", images, n_threads=1) raw_large = False - self.assertEqual(raw_large,output) + self.assertEqual(raw_large, output) images = both_image_types["jpg"] images = sorted(images) output = e2t.process_camera(json_mode, "jpg", images, n_threads=1) jpg_large = copy.deepcopy(self.original_jpg_json) - self.assertDictEqual(jpg_large,output) + self.assertDictEqual(jpg_large, output) - resized_json= eval(open(file_path_resized).read()) + resized_json = eval(open(file_path_resized).read()) self.assertDictEqual(resized_json, resized_test_json) - original_json=eval(open(file_path).read()) + original_json = eval(open(file_path).read()) self.assertDictEqual(original_json, test_json) raw_original_json = eval(open(file_path_raw).read()) self.assertDictEqual(raw_original_json, raw_test_json) def test_resize_mode(self): - no_resize = copy.deepcopy(self.camera_both) - no_resize['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) - no_resize['EXPT_END'] = "now" - no_resize = e2t.CameraFields(no_resize) - self.wipe_output() - for ext,images in e2t.find_image_files(no_resize).items(): + no_resize = copy.deepcopy(self.camera_both) + no_resize['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) + no_resize['EXPT_END'] = "now" + no_resize = e2t.CameraFields(no_resize) + self.wipe_output() + for ext, images in e2t.find_image_files(no_resize).items(): images = sorted(images) output = e2t.process_camera(no_resize, ext, images, n_threads=1) - self.assertEqual(False,output) - # Check the Mini-Jsons - # First the original value - file_path = os.path.join(no_resize.destination, no_resize.ts_structure.format( + self.assertEqual(False, output) + # Check the Mini-Jsons + # First the original value + file_path = os.path.join(no_resize.destination, no_resize.ts_structure.format( folder='originals', res='fullres', step='orig'), no_resize.userfriendlyname - + '-ts-info.json') - original_json = eval(open(file_path).read()) - self.assertDictEqual(original_json, self.original_single_jpg_json) - file_path_resized = os.path.join(no_resize.destination, no_resize.ts_structure.format( + + '-ts-info.json') + original_json = eval(open(file_path).read()) + self.assertDictEqual(original_json, self.original_single_jpg_json) + file_path_resized = os.path.join(no_resize.destination, no_resize.ts_structure.format( folder='outputs', res='1920', step='orig'), no_resize.userfriendlyname - + '-ts-info.json') - self.assertFalse(os.path.exists(file_path_resized)) - - resize = copy.deepcopy(self.camera_both) - resize ['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) - resize ['EXPT_END']="now" - resize ['METHOD'] = 'resize' - resize ['RESOLUTIONS'] = 'original~1920' - resize ['SOURCE'] = os.path.join('.', 'test', 'out', 'timestreams') - resize = e2t.CameraFields(resize) - for ext,images in e2t.find_image_files(resize).items(): + + '-ts-info.json') + self.assertFalse(os.path.exists(file_path_resized)) + + resize = copy.deepcopy(self.camera_both) + resize['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) + resize['EXPT_END'] = "now" + resize['METHOD'] = 'resize' + resize['RESOLUTIONS'] = 'original~1920' + resize['SOURCE'] = os.path.join('.', 'test', 'out', 'timestreams') + resize = e2t.CameraFields(resize) + for ext, images in e2t.find_image_files(resize).items(): images = sorted(images) output = e2t.process_camera(resize, ext, images, n_threads=1) - self.assertEqual(False,output) - self.assertTrue(os.path.exists(file_path_resized)) - - resized_json= eval(open(file_path_resized).read()) + self.assertEqual(False, output) + self.assertTrue(os.path.exists(file_path_resized)) - resized_test_json =self.resized_jpg_json - self.assertDictEqual(resized_json, resized_test_json) - new = Image.open(self.r_resize_path).size - self.assertEqual(new[0], 1920) - self.assertEqual(new[1], 1280) + resized_json = eval(open(file_path_resized).read()) + resized_test_json = self.resized_jpg_json + self.assertDictEqual(resized_json, resized_test_json) + new = Image.open(self.r_resize_path).size + self.assertEqual(new[0], 1920) + self.assertEqual(new[1], 1280) def test_rotate_mode(self): no_rotate = copy.deepcopy(self.camera_both) no_rotate['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) - no_rotate['EXPT_END']="now" + no_rotate['EXPT_END'] = "now" no_rotate = e2t.CameraFields(no_rotate) self.wipe_output() - for ext,images in e2t.find_image_files(no_rotate).items(): + for ext, images in e2t.find_image_files(no_rotate).items(): images = sorted(images) output = e2t.process_camera(no_rotate, ext, images, n_threads=1) - self.assertEqual(False,output) - # Check the Mini-Jsons + self.assertEqual(False, output) + # Check the Mini-Jsons # First the original value file_path = os.path.join(no_rotate.destination, no_rotate.ts_structure.format( folder='originals', res='fullres', step='orig'), no_rotate.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') original_json = eval(open(file_path).read()) test_json = self.original_single_jpg_json self.assertDictEqual(original_json, test_json) file_path_resized = os.path.join(no_rotate.destination, no_rotate.ts_structure.format( folder='outputs', res='1920', step='orig'), no_rotate.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') old = Image.open(self.r_fullres_path).size self.assertEqual(old[0], 5184) @@ -888,20 +877,20 @@ def test_rotate_mode(self): rotate = copy.deepcopy(self.camera_both) rotate['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) - rotate['EXPT_END']="now" + rotate['EXPT_END'] = "now" rotate['METHOD'] = 'rotate' rotate['ORIENTATION'] = 90 rotate['SOURCE'] = os.path.join('.', 'test', 'out', 'timestreams') rotate = e2t.CameraFields(rotate) - for ext,images in e2t.find_image_files(rotate).items(): + for ext, images in e2t.find_image_files(rotate).items(): images = sorted(images) output = e2t.process_camera(rotate, ext, images, n_threads=1) - self.assertEqual(False,output) + self.assertEqual(False, output) new = Image.open(self.r_fullres_path).size - self.assertTrue(abs(old[0]-new[1]) < 2) - self.assertTrue(abs(old[1]-new[0]) < 2) + self.assertTrue(abs(old[0] - new[1]) < 2) + self.assertTrue(abs(old[1] - new[0]) < 2) rotated_json = copy.deepcopy(self.original_single_jpg_json) rotated_json['width'] = 3456 @@ -911,7 +900,6 @@ def test_rotate_mode(self): r_json = eval(open(file_path).read()) self.assertDictEqual(rotated_json, r_json) - def test_structure_format_none(self): ts_format_test = e2t.CameraFields({ 'ARCHIVE_DEST': '/', @@ -938,14 +926,16 @@ def test_structure_format_none(self): 'FILENAME_DATE_MASK': "", 'FN_STRUCTURE': '', 'ORIENTATION': '', - 'DATASETID':1, - 'TIMESHIFT':0, - 'USERFRIENDLYNAME':'', - 'JSON_UPDATES':'' + 'DATASETID': 1, + 'TIMESHIFT': 0, + 'USERFRIENDLYNAME': '', + 'JSON_UPDATES': '' }) - output= (e2t.parse_structures(ts_format_test)) - self.assertEqual(os.path.join("BVZ00000", "EUC-R01C01-C01-F01", "{folder}", "BVZ00000-EUC-R01C01-C01-F01~{res}-{step}"), output.ts_structure) + output = (e2t.parse_structures(ts_format_test)) + self.assertEqual( + os.path.join("BVZ00000", "EUC-R01C01-C01-F01", "{folder}", "BVZ00000-EUC-R01C01-C01-F01~{res}-{step}"), + output.ts_structure) self.assertEqual("BVZ00000-EUC-R01C01-C01-F01~{res}-{step}", output.fn_structure) self.assertEqual('BVZ00000-EUC-R01C01-C01-F01', output.userfriendlyname) @@ -969,17 +959,18 @@ def test_resolution_calc(self): 'SUNSET': "2200", 'USE': True, 'USER': 'Glasshouses', - 'TS_STRUCTURE': os.path.sep.join(['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), + 'TS_STRUCTURE': os.path.sep.join( + ['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), 'PROJECT_OWNER': '', - 'FILENAME_DATE_MASK':'', + 'FILENAME_DATE_MASK': '', 'FN_PARSE': '', 'FN_STRUCTURE': 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}', 'ORIENTATION': '', - 'TIMESHIFT':'', - 'DATASETID':'1', - 'JSON_UPDATES':'', - 'LARGE_JSON':'', - 'USERFRIENDLYNAME':'BVZ00000-EUC-R01C01-C01-F01' + 'TIMESHIFT': '', + 'DATASETID': '1', + 'JSON_UPDATES': '', + 'LARGE_JSON': '', + 'USERFRIENDLYNAME': 'BVZ00000-EUC-R01C01-C01-F01' }) dimensions = e2t.resolution_calc(res_calc, self.jpg_testfile) @@ -1004,17 +995,18 @@ def test_resolution_calc(self): 'SUNSET': "2200", 'USE': True, 'USER': 'Glasshouses', - 'TS_STRUCTURE': os.path.sep.join(['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), + 'TS_STRUCTURE': os.path.sep.join( + ['BVZ00000', 'EUC-R01C01-C01-F01', '{folder}', 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}']), 'PROJECT_OWNER': '', - 'FILENAME_DATE_MASK':'', + 'FILENAME_DATE_MASK': '', 'FN_PARSE': '', 'FN_STRUCTURE': 'BVZ00000-EUC-R01C01-C01-F01~{res}-{step}', 'ORIENTATION': '90', - 'TIMESHIFT':'', - 'DATASETID':'1', - 'JSON_UPDATES':'', - 'LARGE_JSON':'', - 'USERFRIENDLYNAME':'BVZ00000-EUC-R01C01-C01-F01' + 'TIMESHIFT': '', + 'DATASETID': '1', + 'JSON_UPDATES': '', + 'LARGE_JSON': '', + 'USERFRIENDLYNAME': 'BVZ00000-EUC-R01C01-C01-F01' }) dimensions_r = e2t.resolution_calc(res_calc_r, self.jpg_testfile) @@ -1022,59 +1014,59 @@ def test_resolution_calc(self): def test_structure_format_all(self): ts_format_test = e2t.CameraFields( - { - 'ARCHIVE_DEST': '/', - 'EXPT': 'BVZ00000', - 'DESTINATION': '/', - 'CAM_NUM': 1, - 'EXPT_END': '2013_12_31', - 'EXPT_START': '2013_11_01', - 'LOCATION': 'EUC-R01C01', - 'METHOD': 'copy', - 'INTERVAL': '5', - 'IMAGE_TYPES': 'raw~jpg', - 'MODE': 'batch', - 'SOURCE': '/', - 'USE': '1', - 'TS_STRUCTURE': os.path.join('EXPT','LOCATION-location','potato'), - 'FN_STRUCTURE': os.path.join('EXPT', 'LOCATION-location' ,'potato'), - 'USERFRIENDLYNAME': os.path.join('EXPT','LOCATION-location','potato'), - } + { + 'ARCHIVE_DEST': '/', + 'EXPT': 'BVZ00000', + 'DESTINATION': '/', + 'CAM_NUM': 1, + 'EXPT_END': '2013_12_31', + 'EXPT_START': '2013_11_01', + 'LOCATION': 'EUC-R01C01', + 'METHOD': 'copy', + 'INTERVAL': '5', + 'IMAGE_TYPES': 'raw~jpg', + 'MODE': 'batch', + 'SOURCE': '/', + 'USE': '1', + 'TS_STRUCTURE': os.path.join('EXPT', 'LOCATION-location', 'potato'), + 'FN_STRUCTURE': os.path.join('EXPT', 'LOCATION-location', 'potato'), + 'USERFRIENDLYNAME': os.path.join('EXPT', 'LOCATION-location', 'potato'), + } ) - output= (e2t.parse_structures(ts_format_test)) + output = (e2t.parse_structures(ts_format_test)) self.assertEqual(os.path.join("BVZ00000", "EUC-R01C01-location", "potato~{res}-{step}"), output.ts_structure) self.assertEqual(''.join(["BVZ00000", "EUC-R01C01-location", "potato~{res}-{step}"]), output.fn_structure) self.assertEqual(''.join(["BVZ00000", "EUC-R01C01-location", "potato"]), output.userfriendlyname) def test_small_json_write_over(self): small_json = copy.deepcopy(self.camera_both) - small_json['EXPT_START']= "2002_01_01" - small_json['EXPT_END']= "now" - small_json['DESTINATION']=os.path.sep.join(['.', 'test', 'out', 'timestreams']) + small_json['EXPT_START'] = "2002_01_01" + small_json['EXPT_END'] = "now" + small_json['DESTINATION'] = os.path.sep.join(['.', 'test', 'out', 'timestreams']) small_json['METHOD'] = 'move' small_json = e2t.CameraFields(small_json) self.wipe_output() - for ext,images in e2t.find_image_files(small_json).items(): - images = sorted(images) - output = e2t.process_camera(small_json, ext, images, n_threads=1) - self.assertEqual(False,output) - # Check the Mini-Jsons + for ext, images in e2t.find_image_files(small_json).items(): + images = sorted(images) + output = e2t.process_camera(small_json, ext, images, n_threads=1) + self.assertEqual(False, output) + # Check the Mini-Jsons # First the original value file_path = os.path.join(small_json.destination, small_json.ts_structure.format( folder='originals', res='fullres', step='orig'), small_json.userfriendlyname - + '-ts-info.json') + + '-ts-info.json') original_json = eval(open(file_path).read()) test_json = copy.deepcopy(self.original_single_jpg_json) self.assertDictEqual(original_json, test_json) small_json.method = 'resize' e2t.create_small_json("fullres", small_json, (5184, 3456), (1920, 1280), - time.strptime("19990101", "%Y%m%d"), time.strptime("20150101", "%Y%m%d"), - "now", "orig", "http://phenocam.anu.edu.au/cloud/a_data{}/{}".format( - small_json.destination.split("a_data")[-1],small_json.ts_structure if small_json.ts_structure - else small_json.location).replace("\\","/"), [], 'orig') + time.strptime("19990101", "%Y%m%d"), time.strptime("20150101", "%Y%m%d"), + "now", "orig", "http://phenocam.anu.edu.au/cloud/a_data{}/{}".format( + small_json.destination.split("a_data")[-1], small_json.ts_structure if small_json.ts_structure + else small_json.location).replace("\\", "/"), [], 'orig') original_json = eval(open(file_path).read()) - after_json= copy.deepcopy(self.original_single_jpg_json) + after_json = copy.deepcopy(self.original_single_jpg_json) after_json['posix_start'] = 915148800.0 after_json['width'] = 1920 after_json['height'] = 1280 @@ -1107,7 +1099,7 @@ def test_ListByTime(self): def test_DelByTime(self): self.wipe_output() del_time = copy.deepcopy(self.config_list_delete) - del_time= lbt.CameraFields(del_time) + del_time = lbt.CameraFields(del_time) for ext, images in dbt.find_image_files(del_time).items(): dbt.process_timestream(del_time, ext, sorted(images), 1) images_del = sorted(os.listdir(os.path.join(del_time.delete_dest, 'DateCheck'))) @@ -1145,7 +1137,7 @@ def test_sub_folder(self): 'whroo2013_11_10_10_59_59M.jpg', os.path.join('subfolder', "whroo2015_11_12_12_01_01M.jpg") ]) - ] + ] } got = e2t.find_image_files(sub_included) self.assertListEqual(sorted(got['jpg']), expt['jpg']) @@ -1168,10 +1160,11 @@ def test_sub_folder(self): 'whroo2013_11_12_11_01_01M.jpg', 'whroo2013_11_12_12_01_01M.jpg' ]) - ] + ] } self.assertListEqual(sorted(no_subfolder['jpg']), no_sub_expt['jpg']) + if __name__ == "__main__": runner = unittest.TextTestRunner(verbosity=3) unittest.main(testRunner=runner)