diff --git a/exif2timestream.py b/exif2timestream.py index 010e604..37c2ba1 100644 --- a/exif2timestream.py +++ b/exif2timestream.py @@ -1,6 +1,5 @@ from __future__ import print_function from csv import reader, DictReader -import exifread as er import os from os import path import shutil @@ -10,14 +9,23 @@ from itertools import cycle from inspect import isclass import logging - +import re +import pexif +import exifread as er +import warnings +SKIMAGE = False +try: + import skimage + SKIMAGE = True +except ImportError: + pass # versioneer from _version import get_versions __version__ = get_versions()['version'] del get_versions - EXIF_DATE_TAG = "Image DateTime" EXIF_DATE_FMT = "%Y:%m:%d %H:%M:%S" +EXIF_DATE_MASK = EXIF_DATE_FMT TS_V1_FMT = ("%Y/%Y_%m/%Y_%m_%d/%Y_%m_%d_%H/" "{tsname:s}_%Y_%m_%d_%H_%M_%S_{n:02d}.{ext:s}") TS_V2_FMT = ("%Y/%Y_%m/%Y_%m_%d/%Y_%m_%d_%H/" @@ -32,7 +40,7 @@ DATE_NOW_CONSTANTS = {"now", "current"} CLI_OPTS = """ USAGE: - exif2timestream.py [-t PROCESSES -1 -d -l LOGDIR] -c CAM_CONFIG_CSV + exif2timestream.py [-t PROCESSES -1 -d -l LOGDIR -m MASK] -c CAM_CONFIG_CSV exif2timestream.py -g CAM_CONFIG_CSV exif2timestream.py -V @@ -44,6 +52,7 @@ -c CAM_CONFIG_CSV Path to CSV camera config file for normal operation. -g CAM_CONFIG_CSV Generate a template camera configuration file at given path. + -m MASK Mask to Use for parsing dates from filenames -V Print version information. """ @@ -115,11 +124,13 @@ def d2s(date): else: return date + def validate_camera(camera): """Validates and converts to python types the given camera dict (which normally has string values). """ log = logging.getLogger("exif2timestream") + def date(x): if isinstance(x, struct_time): return x @@ -189,7 +200,7 @@ def image_type_str(x): raise ValueError types = x.lower().strip().split('~') for type in types: - if not type in IMAGE_TYPE_CONSTANTS: + if type not in IMAGE_TYPE_CONSTANTS: raise ValueError return types @@ -201,7 +212,7 @@ def __init__(self, valid_values): self.valid_values = set(valid_values) def __call__(self, x): - if not x in self.valid_values: + if x not in self.valid_values: raise ValueError return x @@ -236,18 +247,112 @@ def __call__(self, x): return None +def resize_img(filename, to_width): + # Open the Image and get its width + if not(SKIMAGE): + warnings.warn( + "Skimage Not Installed, Unable to Test Resize", ImportWarning) + return None + img = skimage.io.imread(filename) + w = skimate.novice.open(filename).width + scale = float(to_width) / w + # Rescale the image + img = skimage.transform.rescale(img, scale) + # read in old exxif data + exif_source = pexif.JpegFile.fromFile(filename) + # Save image + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + skimage.io.imsave(filename, img) + # Write new exif data from old image + try: + exif_dest = pexif.JpegFile.fromFile(filename) + exif_dest.exif.primary.ExtendedEXIF.DateTimeOriginal = \ + exif_source.exif.primary.ExtendedEXIF.DateTimeOriginal + exif_dest.writeFile(filename) + except AttributeError: + pass + + +def get_time_from_filename(filename, mask=EXIF_DATE_MASK): + # Replace the year with the regex equivalent to parse + regex_mask = mask.replace("%Y", "\d{4}").replace( + "%m", "\d{2}").replace("%d", "\d{2}") + regex_mask = regex_mask.replace("%H", "\d{2}").replace( + "%M", "\d{2}").replace("%S", "\d{2}") + # Wildcard character before and after the regex + regex_mask = "\.*" + regex_mask + "\.*" + # compile the regex + date_reg_exp = re.compile(regex_mask) + # get the list of possible date matches + matches_list = date_reg_exp.findall(filename) + for match in matches_list: + try: + # Parse each match into a datetime + datetime = strptime(match, mask) + # Return the datetime + return datetime + # If we cant convert it to the date, then go to the next item on the + # list + except ValueError: + continue + # If we cant match anything, then return None + return None + + +def write_exif_date(filename, date_time): + try: + # Read in the file + img = pexif.JpegFile.fromFile(filename) + # Edit the exif data + img.exif.primary.ExtendedEXIF.DateTimeOriginal = strftime( + EXIF_DATE_FMT, date_time) + # Write to the file + img.writeFile(filename) + return True + except IOError: + return False + + def get_file_date(filename, round_secs=1): """ Gets a time.struct_time from an image's EXIF, or None if not possible. """ log = logging.getLogger("exif2timestream") - with open(filename, "rb") as fh: - exif_tags = er.process_file(fh, details=False, stop_tag=EXIF_DATE_TAG) + # Now uses Pexif + try: - str_date = exif_tags[EXIF_DATE_TAG].values + exif_tags = pexif.JpegFile.fromFile(filename) + str_date = exif_tags.exif.primary.ExtendedEXIF.DateTimeOriginal date = strptime(str_date, EXIF_DATE_FMT) - except KeyError: - return None + # print (date) + except AttributeError: + # Try and Grab datetime from the filename + # Grab only the filename, not the directory + shortfilename = os.path.basename(filename) + log.debug("No Exif data in '{0:s}', attempting to read from filename".format( + shortfilename)) + # Try and grab the date + # We can put a custom mask in here if we want + date = get_time_from_filename(filename) + if date is None: + log.debug( + "Unable to scrape date from '{0:s}'".format(shortfilename)) + return None + else: + if not(write_exif_date(filename, date)): + log.debug("Unable to write Exif Data") + return None + return date + except pexif.JpegFile.InvalidFile: + with open(filename, "rb") as fh: + exif_tags = er.process_file( + fh, details=False, stop_tag=EXIF_DATE_TAG) + try: + str_date = exif_tags[EXIF_DATE_TAG].values + date = strptime(str_date, EXIF_DATE_FMT) + except KeyError: + return None if round_secs > 1: date = round_struct_time(date, round_secs) log.debug("Date of '{0:s}' is '{1:s}'".format(filename, d2s(date))) @@ -318,6 +423,8 @@ def timestreamise_image(image, camera, subsec=0, step="orig"): log = logging.getLogger("exif2timestream") # make new image path image_date = get_file_date(image, camera[FIELDS["interval"]] * 60) + # Resize the Image + # resize(image, 1000) if not image_date: log.warn("Couldn't get date for image {}".format(image)) raise SkipImage @@ -517,9 +624,9 @@ def find_image_files(camera): for dir in dirs: if dir.lower() not in IMAGE_SUBFOLDERS and \ not dir.startswith("_"): - log.error("Souce directory has too many subdirs.A") + log.error("Source directory has too many subdirs.") # TODO: Is raising here a good idea? - #raise ValueError("too many subdirs") + raise ValueError("too many subdirs") for fle in files: this_ext = path.splitext(fle)[-1].lower().strip(".") if this_ext == ext or ext == "raw" and this_ext in RAW_FORMATS: @@ -581,6 +688,9 @@ def main(opts): # beginneth the actual main loop start_time = time() cameras = parse_camera_config_csv(opts["-c"]) + global EXIF_DATE_MASK # Needed below + if opts['-m'] is not None: + EXIF_DATE_MASK = opts["-m"] n_images = 0 for camera in cameras: msg = "Processing experiment {}, location {}\n".format( diff --git a/run_from_cmdline b/run_from_cmdline index b936b02..37534a8 100755 --- a/run_from_cmdline +++ b/run_from_cmdline @@ -9,4 +9,4 @@ mkdir -p test/out/archive mkdir -p test/out/timestreams python -m exif2timestream -l log -d -c ./test/config.csv if [ -n "$(which tree)" ] ; then tree test/out; fi -rm -r test/out +# rm -r test/out diff --git a/setup.py b/setup.py index 4d93078..81fe23b 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ "ExifRead==1.4.2", "docopt==0.6.1", "voluptuous==0.8.4", + "pexif==0.15", ] test_requires = [ diff --git a/test/test_exif2timestream.py b/test/test_exif2timestream.py index 28f60da..3146318 100644 --- a/test/test_exif2timestream.py +++ b/test/test_exif2timestream.py @@ -9,6 +9,14 @@ import unittest from voluptuous import MultipleInvalid from tempfile import NamedTemporaryFile +import pexif +import warnings +SKIMAGE = False +try: + import skimage + # SKIMAGE = True +except ImportError: + pass class TestExifTraitcapture(unittest.TestCase): @@ -225,7 +233,8 @@ def test_find_image_files(self): 'jpg/IMG_0001.JPG', 'jpg/IMG_0002.JPG', 'jpg/IMG_0630.JPG', - 'jpg/IMG_0633.JPG'] + 'jpg/IMG_0633.JPG', + 'jpg/whroo20131104_020255M.jpg'] }, "raw": {path.join(self.camupload_dir, 'raw/IMG_0001.CR2')}, } @@ -312,6 +321,47 @@ def test_generate_config_csv(self): e2t.generate_config_csv(out_csv) self._md5test(out_csv, "bf1ff915a42390a15ab8e4704e5c38e9") + # Tests for checking parsing of dates from filename + def test_check_date_parse(self): + got = e2t.get_time_from_filename( + "whroo20141101_001212M.jpg", "%Y%m%d_%H%M%S") + expected = strptime("20141101_001212", "%Y%m%d_%H%M%S") + self.assertEqual(got, expected) + got = e2t.get_time_from_filename("TRN-NC-DSC-01~640_2013_06_01_10_45_00_00.jpg", + "%Y_%m_%d_%H_%M_%S") + expected = strptime("2013_06_01_10_45_00", "%Y_%m_%d_%H_%M_%S") + self.assertEqual(got, expected) + + def test_check_write_exif(self): + # Write To Exif + filename = 'jpg/whroo20131104_020255M.jpg' + date_time = e2t.get_time_from_filename( + path.join(self.camupload_dir, filename), "%Y%m%d_%H%M%S") + e2t.write_exif_date(path.join(self.camupload_dir, filename), date_time) + + # Read From Exif + exif_tags = pexif.JpegFile.fromFile( + path.join(self.camupload_dir, filename)) + str_date = exif_tags.exif.primary.ExtendedEXIF.DateTimeOriginal + date = strptime(str_date, "%Y:%m:%d %H:%M:%S") + + # Check Equal + self.assertEqual(date_time, date) + + # Tests for checking image resizing + def test_check_resize_img(self): + if(SKIMAGE): + filename = 'jpg/whroo20131104_020255M.jpg' + new_width = 400 + e2t.resize_img(path.join(self.camupload_dir, filename), new_width) + img = skimage.io.imread(path.join(self.camupload_dir, filename)) + w = skimage.novice.open( + path.join(self.camupload_dir, filename)).width + self.assertEqual(w, new_width) + else: + warnings.warn( + "Skimage Not Installed, Unable to Test Resize", ImportWarning) + # tests for main function def test_main(self): e2t.main({ @@ -320,6 +370,7 @@ def test_main(self): '-a': None, '-c': self.test_config_csv, '-l': self.out_dirname, + '-m': None, '-g': None, '-t': None}) #os.system("tree %s" % path.dirname(self.out_dirname)) @@ -332,6 +383,7 @@ def test_main_raw(self): '-a': None, '-c': self.test_config_raw_csv, '-l': self.out_dirname, + '-m': None, '-g': None, '-t': None}) #os.system("tree %s" % path.dirname(self.out_dirname)) @@ -345,6 +397,7 @@ def test_main_expt_dates(self): '-a': None, '-c': self.test_config_dates_csv, '-l': self.out_dirname, + '-m': None, '-g': None, '-t': None}) #os.system("tree %s" % path.dirname(self.out_dirname)) @@ -358,6 +411,7 @@ def test_main_threads(self): '-a': None, '-c': self.test_config_csv, '-l': self.out_dirname, + '-m': None, '-g': None, '-t': '2'}) self.assertTrue(path.exists(self.r_fullres_path)) @@ -370,6 +424,7 @@ def test_main_threads_bad(self): '-a': None, '-c': self.test_config_csv, '-l': self.out_dirname, + '-m': None, '-g': None, '-t': "several"}) self.assertTrue(path.exists(self.r_fullres_path)) @@ -382,6 +437,7 @@ def test_main_threads_one(self): '-a': None, '-c': self.test_config_csv, '-l': self.out_dirname, + '-m': None, '-g': None, '-t': None}) self.assertTrue(path.exists(self.r_fullres_path)) @@ -398,6 +454,7 @@ def test_main_generate(self): '-a': None, '-c': None, '-l': self.out_dirname, + '-m': None, '-g': conf_out, '-t': None}) self.assertTrue(path.exists(conf_out)) diff --git a/test/unburnable/camupload/jpg/whroo20131104_020255M.jpg b/test/unburnable/camupload/jpg/whroo20131104_020255M.jpg new file mode 100644 index 0000000..47ff313 Binary files /dev/null and b/test/unburnable/camupload/jpg/whroo20131104_020255M.jpg differ