Skip to content
Permalink
Browse files

Optimize exiftool calls by adding an ExifTool singleton in pyexiftool…

… library (#352)

This fix results in a 10x performance improvement [1] enabling a single exiftool subprocess to elimate spawing exiftool for each image.

Closes #350 #347 

[1] #350 (comment)
  • Loading branch information
amaleki authored and jmathai committed Jan 14, 2020
1 parent 75e6590 commit d8cee15f32d3853070111c972ecf987a05a55549
@@ -30,11 +30,12 @@
from elodie.media.video import Video
from elodie.plugins.plugins import Plugins
from elodie.result import Result

from elodie.external.pyexiftool import ExifTool
from elodie.dependencies import get_exiftool
from elodie import constants

FILESYSTEM = FileSystem()


def import_file(_file, destination, album_from_folder, trash, allow_duplicates):

_file = _decode(_file)
@@ -368,4 +369,10 @@ def main():


if __name__ == '__main__':
main()
#Initialize ExifTool Subprocess
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
with ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs) as et:
main()
@@ -65,6 +65,8 @@
import logging
import codecs

from future.utils import with_metaclass

try: # Py3k compatibility
basestring
except NameError:
@@ -151,8 +153,16 @@ def format_error (result):
else:
return 'exiftool finished with error: "%s"' % strip_nl(result)

class Singleton(type):
"""Metaclass to use the singleton [anti-]pattern"""
instance = None

def __call__(cls, *args, **kwargs):
if cls.instance is None:
cls.instance = super(Singleton, cls).__call__(*args, **kwargs)
return cls.instance

class ExifTool(object):
class ExifTool(object, with_metaclass(Singleton)):
"""Run the `exiftool` command-line tool and communicate to it.
You can pass two arguments to the constructor:
@@ -19,7 +19,6 @@
from elodie.media.base import Base, get_all_subclasses
from elodie.plugins.plugins import Plugins


class FileSystem(object):
"""A class for interacting with the file system."""

@@ -48,7 +47,6 @@ def __init__(self):
# Instantiate a plugins object
self.plugins = Plugins()


def create_directory(self, directory_path):
"""Create a directory if it does not already exist.
@@ -644,4 +642,4 @@ def should_exclude(self, path, regex_list=set(), needs_compiled=False):
compiled_list.append(re.compile(regex))
regex_list = compiled_list

return any(regex.search(path) for regex in regex_list)
return any(regex.search(path) for regex in regex_list)
@@ -13,12 +13,9 @@
import os

# load modules
from elodie import constants
from elodie.dependencies import get_exiftool
from elodie.external.pyexiftool import ExifTool
from elodie.media.base import Base


class Media(Base):

"""The base class for all media objects.
@@ -52,10 +49,7 @@ def __init__(self, source=None):
self.longitude_ref_key = 'EXIF:GPSLongitudeRef'
self.original_name_key = 'XMP:OriginalFileName'
self.set_gps_ref = True
self.exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
self.exif_metadata = None

def get_album(self):
"""Get album from EXIF
@@ -122,16 +116,15 @@ def get_exiftool_attributes(self):
:returns: dict, or False if exiftool was not available.
"""
source = self.source
exiftool = get_exiftool()
if(exiftool is None):
return False

with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et:
metadata = et.get_metadata(source)
if not metadata:
return False
#Cache exif metadata results and use if already exists for media
if(self.exif_metadata is None):
self.exif_metadata = ExifTool().get_metadata(source)

if not self.exif_metadata:
return False

return metadata
return self.exif_metadata

def get_camera_make(self):
"""Get the camera make stored in EXIF.
@@ -211,6 +204,7 @@ def reset_cache(self):
"""Resets any internal cache
"""
self.exiftool_attributes = None
self.exif_metadata = None
super(Media, self).reset_cache()

def set_album(self, album):
@@ -318,13 +312,8 @@ def __set_tags(self, tags):
return None

source = self.source
exiftool = get_exiftool()
if(exiftool is None):
return False


status = ''
with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et:
status = et.set_tags(tags, source)
status = ExifTool().set_tags(tags,source)

return status != ''
@@ -20,9 +20,21 @@
from elodie.media.photo import Photo
from elodie.media.video import Video
from nose.plugins.skip import SkipTest
from elodie.external.pyexiftool import ExifTool
from elodie.dependencies import get_exiftool
from elodie import constants

os.environ['TZ'] = 'GMT'

def setup_module():
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()

def teardown_module():
ExifTool().terminate

def test_create_directory_success():
filesystem = FileSystem()
@@ -575,7 +587,7 @@ def test_get_folder_path_with_with_more_than_two_levels():
path = filesystem.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config

assert path == os.path.join('2015','12','Sunnyvale, California'), path

@mock.patch('elodie.config.config_file', '%s/config.ini-location-date' % gettempdir())
@@ -15,6 +15,8 @@
from datetime import timedelta

from elodie.compatability import _rename
from elodie.external.pyexiftool import ExifTool
from elodie.dependencies import get_exiftool
from elodie import constants

def checksum(file_path, blocksize=65536):
@@ -159,3 +161,14 @@ def restore_dbs():
# This is no longer needed. See gh-322
# https://github.com/jmathai/elodie/issues/322
pass


def setup_module():
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()

def teardown_module():
ExifTool().terminate
@@ -16,9 +16,22 @@
from elodie.media.media import Media
from elodie.media.video import Video
from elodie.media.audio import Audio
from elodie.external.pyexiftool import ExifTool
from elodie.dependencies import get_exiftool
from elodie import constants

os.environ['TZ'] = 'GMT'

def setup_module():
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()

def teardown_module():
ExifTool().terminate

def test_audio_extensions():
audio = Audio()
extensions = audio.extensions
@@ -23,6 +23,8 @@

os.environ['TZ'] = 'GMT'

setup_module = helper.setup_module
teardown_module = helper.teardown_module

def test_get_all_subclasses():
subclasses = get_all_subclasses(Base)
@@ -21,6 +21,8 @@

os.environ['TZ'] = 'GMT'

setup_module = helper.setup_module
teardown_module = helper.teardown_module

def test_get_file_path():
media = Media(helper.get_file('plain.jpg'))
@@ -86,7 +88,7 @@ def test_get_original_name_invalid_file():
original_name = media.get_original_name()

assert original_name is None, original_name

def test_set_original_name_when_exists():
temporary_folder, folder = helper.create_working_folder()

@@ -20,6 +20,9 @@

os.environ['TZ'] = 'GMT'

setup_module = helper.setup_module
teardown_module = helper.teardown_module

def test_photo_extensions():
photo = Photo()
extensions = photo.extensions
@@ -30,9 +30,8 @@
secrets_file
)

sample_photo = Photo(helper.get_file('plain.jpg'))
sample_metadata = sample_photo.get_metadata()
sample_metadata['original_name'] = 'foobar'
setup_module = helper.setup_module
teardown_module = helper.teardown_module

@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-set-session' % gettempdir())
def test_googlephotos_set_session():
@@ -57,6 +56,9 @@ def test_googlephotos_after_supported():
if hasattr(load_config, 'config'):
del load_config.config

sample_photo = Photo(helper.get_file('plain.jpg'))
sample_metadata = sample_photo.get_metadata()
sample_metadata['original_name'] = 'foobar'
final_file_path = helper.get_file('plain.jpg')
gp = GooglePhotos()
gp.after('', '', final_file_path, sample_metadata)
@@ -162,6 +164,9 @@ def test_googlephotos_batch():
if hasattr(load_config, 'config'):
del load_config.config

sample_photo = Photo(helper.get_file('plain.jpg'))
sample_metadata = sample_photo.get_metadata()
sample_metadata['original_name'] = 'foobar'
final_file_path = helper.get_file('plain.jpg')
gp = GooglePhotos()
gp.after('', '', final_file_path, sample_metadata)

0 comments on commit d8cee15

Please sign in to comment.
You can’t perform that action at this time.