From ffdaf4ec86d2d55191f0a5778c5e39e60d60e639 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Wed, 10 Jul 2019 03:59:50 -0700 Subject: [PATCH 01/16] Use PIL(pillow) as a fallback mechanism image format identification and add HEIC support (#320) Fixes #281 and #269 When elodie imports images, imghdr.what is used to determine the image type. imghdr implementation won't support all JPEG variants. Because of this, even for a valid JPEG file, imghdr.what returns None causing elodie to skip the image during import. This commit adds a fallback mechanism which uses PIL(pillow) library to identify image formats. When imghdr fails, elodie uses pillow to identify image format. When pillow is unavailable, it will be treated as an invalid media file. Pillow is imported only when imghdr fails. --- elodie/media/media.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elodie/media/media.py b/elodie/media/media.py index 1eca7320..85bc1487 100644 --- a/elodie/media/media.py +++ b/elodie/media/media.py @@ -128,6 +128,7 @@ def get_exiftool_attributes(self): with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et: metadata = et.get_metadata(source) + print(metadata) if not metadata: return False From 76c85f3836e291f445cc4878276a47ea338d5ef8 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Wed, 26 Jun 2019 00:09:35 -0700 Subject: [PATCH 02/16] Initial functional code to upload photo to google photos --- elodie/plugins/googlephotos/googlephotos.py | 63 +++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 elodie/plugins/googlephotos/googlephotos.py diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py new file mode 100644 index 00000000..20f24a60 --- /dev/null +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -0,0 +1,63 @@ +import json + +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import AuthorizedSession +from google.oauth2.credentials import Credentials + +scopes = [ +"https://www.googleapis.com/auth/photoslibrary", +"https://www.googleapis.com/auth/photoslibrary.appendonly", +"https://www.googleapis.com/auth/photoslibrary.sharing" +] + +auth_file = 'client_id.json' + +try: + creds = Credentials.from_authorized_user_file(auth_file, scopes) +except: + print('no creds') + flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-5g51kj0feetbet88o8le5i16hbr3ucb6.apps.googleusercontent.com-3.json', scopes) + #creds = tools.run_flow(flow, store) + creds = flow.run_local_server(host='localhost', + port=8080, + authorization_prompt_message="", + success_message='The auth flow is complete; you may close this window.', + open_browser=True) + cred_dict = { + 'token': creds.token, + 'refresh_token': creds.refresh_token, + 'id_token': creds.id_token, + 'scopes': creds.scopes, + 'token_uri': creds.token_uri, + 'client_id': creds.client_id, + 'client_secret': creds.client_secret + } + + with open(auth_file, 'w') as f: + f.write(json.dumps(cred_dict)) + +session = AuthorizedSession(creds) + + +"""google_photos = build('photoslibrary', 'v1', http=creds.authorize(Http())) +results = google_photos.mediaItems().list().execute() +items = results.get('mediaItems', []) +print(items)""" + +session.headers["Content-type"] = "application/octet-stream" +session.headers["X-Goog-Upload-Protocol"] = "raw" +session.headers["X-Goog-Upload-File-Name"] = 'foo.jpg' #os.path.basename(photo_file_name) + +photo_file = open("/Users/jaisen/Downloads/test.png", mode='rb') +photo_bytes = photo_file.read() + +print(photo_bytes) + +upload_token = session.post('https://photoslibrary.googleapis.com/v1/uploads', photo_bytes) +if (upload_token.status_code == 200) and (upload_token.content): + create_body = json.dumps({"newMediaItems":[{"description":"","simpleMediaItem":{"uploadToken":upload_token.content.decode()}}]}, indent=4) + resp = session.post('https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate', create_body).json() + print(resp) + if "newMediaItemResults" in resp: + status = resp["newMediaItemResults"][0]["status"] + print(status) From 96166ee85f23759719207b0ca6427bae6acedc40 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Fri, 28 Jun 2019 00:12:43 -0700 Subject: [PATCH 03/16] Working plugin module with dummy plugin and tests passing --- elodie/plugins/googlephotos/__init__.py | 0 elodie/plugins/googlephotos/googlephotos.py | 33 ++++++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) create mode 100644 elodie/plugins/googlephotos/__init__.py diff --git a/elodie/plugins/googlephotos/__init__.py b/elodie/plugins/googlephotos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 20f24a60..48c3ed7e 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -1,3 +1,18 @@ +""" +Plugin object. + +.. moduleauthor:: Jaisen Mathai +""" +from __future__ import print_function +from builtins import object + + +class GooglePhotos(object): + """A class to execute plugin actions.""" + pass + +""" + import json from google_auth_oauthlib.flow import InstalledAppFlow @@ -16,13 +31,9 @@ creds = Credentials.from_authorized_user_file(auth_file, scopes) except: print('no creds') - flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-5g51kj0feetbet88o8le5i16hbr3ucb6.apps.googleusercontent.com-3.json', scopes) - #creds = tools.run_flow(flow, store) - creds = flow.run_local_server(host='localhost', - port=8080, - authorization_prompt_message="", - success_message='The auth flow is complete; you may close this window.', - open_browser=True) + #flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-5g51kj0feetbet88o8le5i16hbr3ucb6.apps.googleusercontent.com-3.json', scopes) + flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com.json', scopes) + creds = flow.run_local_server() cred_dict = { 'token': creds.token, 'refresh_token': creds.refresh_token, @@ -39,10 +50,6 @@ session = AuthorizedSession(creds) -"""google_photos = build('photoslibrary', 'v1', http=creds.authorize(Http())) -results = google_photos.mediaItems().list().execute() -items = results.get('mediaItems', []) -print(items)""" session.headers["Content-type"] = "application/octet-stream" session.headers["X-Goog-Upload-Protocol"] = "raw" @@ -51,8 +58,6 @@ photo_file = open("/Users/jaisen/Downloads/test.png", mode='rb') photo_bytes = photo_file.read() -print(photo_bytes) - upload_token = session.post('https://photoslibrary.googleapis.com/v1/uploads', photo_bytes) if (upload_token.status_code == 200) and (upload_token.content): create_body = json.dumps({"newMediaItems":[{"description":"","simpleMediaItem":{"uploadToken":upload_token.content.decode()}}]}, indent=4) @@ -60,4 +65,4 @@ print(resp) if "newMediaItemResults" in resp: status = resp["newMediaItemResults"][0]["status"] - print(status) + print(status)""" From 2bdb76fb572b534904a2c96937f071402d2f01e7 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Fri, 28 Jun 2019 00:15:51 -0700 Subject: [PATCH 04/16] clean up --- elodie/plugins/googlephotos/googlephotos.py | 58 +-------------------- 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 48c3ed7e..84ed69d4 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -1,5 +1,5 @@ """ -Plugin object. +Google Photos plugin object. .. moduleauthor:: Jaisen Mathai """ @@ -10,59 +10,3 @@ class GooglePhotos(object): """A class to execute plugin actions.""" pass - -""" - -import json - -from google_auth_oauthlib.flow import InstalledAppFlow -from google.auth.transport.requests import AuthorizedSession -from google.oauth2.credentials import Credentials - -scopes = [ -"https://www.googleapis.com/auth/photoslibrary", -"https://www.googleapis.com/auth/photoslibrary.appendonly", -"https://www.googleapis.com/auth/photoslibrary.sharing" -] - -auth_file = 'client_id.json' - -try: - creds = Credentials.from_authorized_user_file(auth_file, scopes) -except: - print('no creds') - #flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-5g51kj0feetbet88o8le5i16hbr3ucb6.apps.googleusercontent.com-3.json', scopes) - flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com.json', scopes) - creds = flow.run_local_server() - cred_dict = { - 'token': creds.token, - 'refresh_token': creds.refresh_token, - 'id_token': creds.id_token, - 'scopes': creds.scopes, - 'token_uri': creds.token_uri, - 'client_id': creds.client_id, - 'client_secret': creds.client_secret - } - - with open(auth_file, 'w') as f: - f.write(json.dumps(cred_dict)) - -session = AuthorizedSession(creds) - - - -session.headers["Content-type"] = "application/octet-stream" -session.headers["X-Goog-Upload-Protocol"] = "raw" -session.headers["X-Goog-Upload-File-Name"] = 'foo.jpg' #os.path.basename(photo_file_name) - -photo_file = open("/Users/jaisen/Downloads/test.png", mode='rb') -photo_bytes = photo_file.read() - -upload_token = session.post('https://photoslibrary.googleapis.com/v1/uploads', photo_bytes) -if (upload_token.status_code == 200) and (upload_token.content): - create_body = json.dumps({"newMediaItems":[{"description":"","simpleMediaItem":{"uploadToken":upload_token.content.decode()}}]}, indent=4) - resp = session.post('https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate', create_body).json() - print(resp) - if "newMediaItemResults" in resp: - status = resp["newMediaItemResults"][0]["status"] - print(status)""" From b2dce004f40cc3fae0027a443cfd6d33b24c3543 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Fri, 28 Jun 2019 00:25:11 -0700 Subject: [PATCH 05/16] reinclude google photos code --- elodie/plugins/googlephotos/googlephotos.py | 58 ++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 84ed69d4..48c3ed7e 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -1,5 +1,5 @@ """ -Google Photos plugin object. +Plugin object. .. moduleauthor:: Jaisen Mathai """ @@ -10,3 +10,59 @@ class GooglePhotos(object): """A class to execute plugin actions.""" pass + +""" + +import json + +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import AuthorizedSession +from google.oauth2.credentials import Credentials + +scopes = [ +"https://www.googleapis.com/auth/photoslibrary", +"https://www.googleapis.com/auth/photoslibrary.appendonly", +"https://www.googleapis.com/auth/photoslibrary.sharing" +] + +auth_file = 'client_id.json' + +try: + creds = Credentials.from_authorized_user_file(auth_file, scopes) +except: + print('no creds') + #flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-5g51kj0feetbet88o8le5i16hbr3ucb6.apps.googleusercontent.com-3.json', scopes) + flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com.json', scopes) + creds = flow.run_local_server() + cred_dict = { + 'token': creds.token, + 'refresh_token': creds.refresh_token, + 'id_token': creds.id_token, + 'scopes': creds.scopes, + 'token_uri': creds.token_uri, + 'client_id': creds.client_id, + 'client_secret': creds.client_secret + } + + with open(auth_file, 'w') as f: + f.write(json.dumps(cred_dict)) + +session = AuthorizedSession(creds) + + + +session.headers["Content-type"] = "application/octet-stream" +session.headers["X-Goog-Upload-Protocol"] = "raw" +session.headers["X-Goog-Upload-File-Name"] = 'foo.jpg' #os.path.basename(photo_file_name) + +photo_file = open("/Users/jaisen/Downloads/test.png", mode='rb') +photo_bytes = photo_file.read() + +upload_token = session.post('https://photoslibrary.googleapis.com/v1/uploads', photo_bytes) +if (upload_token.status_code == 200) and (upload_token.content): + create_body = json.dumps({"newMediaItems":[{"description":"","simpleMediaItem":{"uploadToken":upload_token.content.decode()}}]}, indent=4) + resp = session.post('https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate', create_body).json() + print(resp) + if "newMediaItemResults" in resp: + status = resp["newMediaItemResults"][0]["status"] + print(status)""" From 9d5953f536cbefe401463742d6b124a707b47c25 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Thu, 4 Jul 2019 03:10:26 -0700 Subject: [PATCH 06/16] rebased master and cleaned up a little --- elodie/plugins/dummy/dummy.py | 1 - elodie/plugins/googlephotos/googlephotos.py | 17 ++++++++++++----- elodie/plugins/plugins.py | 3 +++ elodie/plugins/runtimeerror/runtimeerror.py | 1 - elodie/plugins/throwerror/throwerror.py | 1 - 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/elodie/plugins/dummy/dummy.py b/elodie/plugins/dummy/dummy.py index 6c5eff16..0d218ae7 100644 --- a/elodie/plugins/dummy/dummy.py +++ b/elodie/plugins/dummy/dummy.py @@ -4,7 +4,6 @@ .. moduleauthor:: Jaisen Mathai """ from __future__ import print_function -from builtins import object from elodie.plugins.plugins import PluginBase diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 48c3ed7e..82aff684 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -1,18 +1,24 @@ """ -Plugin object. +Google Photos plugin object. .. moduleauthor:: Jaisen Mathai """ from __future__ import print_function -from builtins import object +from elodie.plugins.plugins import PluginBase -class GooglePhotos(object): +class GooglePhotos(PluginBase): """A class to execute plugin actions.""" - pass -""" + __name__ = 'GooglePhotos' + + def __init__(self): + pass + def before(self, file_path, destination_path, media): + pass + +i""" import json from google_auth_oauthlib.flow import InstalledAppFlow @@ -66,3 +72,4 @@ class GooglePhotos(object): if "newMediaItemResults" in resp: status = resp["newMediaItemResults"][0]["status"] print(status)""" + diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index 5720355f..2e8e103a 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -21,6 +21,9 @@ class PluginBase(object): __name__ = 'PluginBase' + def before(self, file_path, destination_path, media): + pass + def log(self, msg): log.info(msg) diff --git a/elodie/plugins/runtimeerror/runtimeerror.py b/elodie/plugins/runtimeerror/runtimeerror.py index 8468b1c2..d4a486a4 100644 --- a/elodie/plugins/runtimeerror/runtimeerror.py +++ b/elodie/plugins/runtimeerror/runtimeerror.py @@ -4,7 +4,6 @@ .. moduleauthor:: Jaisen Mathai """ from __future__ import print_function -from builtins import object from elodie.plugins.plugins import PluginBase diff --git a/elodie/plugins/throwerror/throwerror.py b/elodie/plugins/throwerror/throwerror.py index 7b44f1d3..0bd74a0d 100644 --- a/elodie/plugins/throwerror/throwerror.py +++ b/elodie/plugins/throwerror/throwerror.py @@ -4,7 +4,6 @@ .. moduleauthor:: Jaisen Mathai """ from __future__ import print_function -from builtins import object from elodie.plugins.plugins import PluginBase, ElodiePluginError From 0135b0371871111a139eef11fb3fc822a3176ffb Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Mon, 8 Jul 2019 23:38:11 -0700 Subject: [PATCH 07/16] Initial google photos plugin with working tests. --- elodie/config.py | 10 ++ elodie/filesystem.py | 8 + elodie/plugins/dummy/dummy.py | 2 +- elodie/plugins/googlephotos/googlephotos.py | 153 +++++++++++------- elodie/plugins/plugins.py | 10 +- elodie/plugins/runtimeerror/runtimeerror.py | 2 +- elodie/plugins/throwerror/throwerror.py | 2 +- .../files/plugins/googlephotos/auth_file.json | 1 + .../plugins/googlephotos/secrets_file.json | 1 + elodie/tests/plugins/googlephotos/__init__.py | 0 .../plugins/googlephotos/googlephotos_test.py | 79 +++++++++ 11 files changed, 206 insertions(+), 62 deletions(-) create mode 100644 elodie/tests/files/plugins/googlephotos/auth_file.json create mode 100644 elodie/tests/files/plugins/googlephotos/secrets_file.json create mode 100644 elodie/tests/plugins/googlephotos/__init__.py create mode 100644 elodie/tests/plugins/googlephotos/googlephotos_test.py diff --git a/elodie/config.py b/elodie/config.py index 4bfb6ef2..24e54be8 100644 --- a/elodie/config.py +++ b/elodie/config.py @@ -27,3 +27,13 @@ def load_plugin_config(): return config['Plugins']['plugins'].split(',') return [] + +def load_config_for_plugin(name): + # Plugins store data using Plugin%PluginName% format. + key = 'Plugin{}'.format(name) + config = load_config() + + if key in config: + return config[key] + + return {} diff --git a/elodie/filesystem.py b/elodie/filesystem.py index f243c206..14731741 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -594,6 +594,14 @@ def process_file(self, _file, destination, media, **kwargs): db.add_hash(checksum, dest_path) db.update_hash_db() + # Run `after()` for every loaded plugin and if any of them raise an exception + # then we skip importing the file and log a message. + plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, media) + if(plugins_run_after_status == False): + log.warn('At least one plugin pre-run failed for %s' % _file) + return + + return dest_path def set_utime_from_metadata(self, metadata, file_path): diff --git a/elodie/plugins/dummy/dummy.py b/elodie/plugins/dummy/dummy.py index 0d218ae7..2ea66f62 100644 --- a/elodie/plugins/dummy/dummy.py +++ b/elodie/plugins/dummy/dummy.py @@ -15,6 +15,6 @@ class Dummy(PluginBase): def __init__(self): self.before_ran = False - def before(self, file_path, destination_path, media): + def before(self, file_path, destination_folder, media): self.before_ran = True diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 82aff684..d059bbd6 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -5,71 +5,110 @@ """ from __future__ import print_function +import json + +from os.path import basename, isfile + +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import AuthorizedSession +from google.oauth2.credentials import Credentials + from elodie.plugins.plugins import PluginBase class GooglePhotos(PluginBase): - """A class to execute plugin actions.""" + """A class to execute plugin actions. + + Requires a config file with the following configurations set. + secrets_file: + The full file path where to find the downloaded secrets. + auth_file: + The full file path where to store authenticated tokens. + + """ __name__ = 'GooglePhotos' def __init__(self): + super(GooglePhotos, self).__init__() + self.upload_url = 'https://photoslibrary.googleapis.com/v1/uploads' + self.media_create_url = 'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate' + self.scopes = [ + 'https://www.googleapis.com/auth/photoslibrary', + 'https://www.googleapis.com/auth/photoslibrary.appendonly', + 'https://www.googleapis.com/auth/photoslibrary.sharing' + ] + + self.secrets_file = None + if('secrets_file' in self.config_for_plugin): + self.secrets_file = self.config_for_plugin['secrets_file'] + # 'client_id.json' + self.auth_file = None + if('auth_file' in self.config_for_plugin): + self.auth_file = self.config_for_plugin['auth_file'] + self.session = None + + def after(self, file_path, destination_folder, final_file_path, media): pass - def before(self, file_path, destination_path, media): + def before(self, file_path, destination_folder, media): pass -i""" -import json - -from google_auth_oauthlib.flow import InstalledAppFlow -from google.auth.transport.requests import AuthorizedSession -from google.oauth2.credentials import Credentials - -scopes = [ -"https://www.googleapis.com/auth/photoslibrary", -"https://www.googleapis.com/auth/photoslibrary.appendonly", -"https://www.googleapis.com/auth/photoslibrary.sharing" -] - -auth_file = 'client_id.json' - -try: - creds = Credentials.from_authorized_user_file(auth_file, scopes) -except: - print('no creds') - #flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-5g51kj0feetbet88o8le5i16hbr3ucb6.apps.googleusercontent.com-3.json', scopes) - flow = InstalledAppFlow.from_client_secrets_file('/Users/jaisen/Downloads/client_secret_1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com.json', scopes) - creds = flow.run_local_server() - cred_dict = { - 'token': creds.token, - 'refresh_token': creds.refresh_token, - 'id_token': creds.id_token, - 'scopes': creds.scopes, - 'token_uri': creds.token_uri, - 'client_id': creds.client_id, - 'client_secret': creds.client_secret - } - - with open(auth_file, 'w') as f: - f.write(json.dumps(cred_dict)) - -session = AuthorizedSession(creds) - - - -session.headers["Content-type"] = "application/octet-stream" -session.headers["X-Goog-Upload-Protocol"] = "raw" -session.headers["X-Goog-Upload-File-Name"] = 'foo.jpg' #os.path.basename(photo_file_name) - -photo_file = open("/Users/jaisen/Downloads/test.png", mode='rb') -photo_bytes = photo_file.read() - -upload_token = session.post('https://photoslibrary.googleapis.com/v1/uploads', photo_bytes) -if (upload_token.status_code == 200) and (upload_token.content): - create_body = json.dumps({"newMediaItems":[{"description":"","simpleMediaItem":{"uploadToken":upload_token.content.decode()}}]}, indent=4) - resp = session.post('https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate', create_body).json() - print(resp) - if "newMediaItemResults" in resp: - status = resp["newMediaItemResults"][0]["status"] - print(status)""" - + def set_session(self): + # Try to load credentials from an auth file. + # If it doesn't exist or is not valid then catch the + # exception and reauthenticate. + try: + creds = Credentials.from_authorized_user_file(self.auth_file, self.scopes) + except: + print(self.secrets_file) + flow = InstalledAppFlow.from_client_secrets_file(self.secrets_file, self.scopes) + creds = flow.run_local_server() + cred_dict = { + 'token': creds.token, + 'refresh_token': creds.refresh_token, + 'id_token': creds.id_token, + 'scopes': creds.scopes, + 'token_uri': creds.token_uri, + 'client_id': creds.client_id, + 'client_secret': creds.client_secret + } + + # Store the returned authentication tokens to the auth_file. + with open(self.auth_file, 'w') as f: + f.write(json.dumps(cred_dict)) + + self.session = AuthorizedSession(creds) + self.session.headers["Content-type"] = "application/octet-stream" + self.session.headers["X-Goog-Upload-Protocol"] = "raw" + + def upload(self, path_to_photo): + self.set_session() + if(self.session is None): + self.log('Could not initialize session') + return None + + self.session.headers["X-Goog-Upload-File-Name"] = basename(path_to_photo) + if(not isfile(path_to_photo)): + self.log('Could not find file: {}'.format(path_to_photo)) + return None + + with open(path_to_photo, 'rb') as f: + photo_bytes = f.read() + + upload_token = self.session.post(self.upload_url, photo_bytes) + if(upload_token.status_code != 200 or not upload_token.content): + self.log('Uploading media failed: ({}) {}'.format(upload_token.status_code, upload_token.content)) + return None + + create_body = json.dumps({'newMediaItems':[{'description':'','simpleMediaItem':{'uploadToken':upload_token.content.decode()}}]}, indent=4) + resp = self.session.post(self.media_create_url, create_body).json() + if( + 'newMediaItemResults' not in resp or + 'status' not in resp['newMediaItemResults'][0] or + 'message' not in resp['newMediaItemResults'][0]['status'] or + resp['newMediaItemResults'][0]['status']['message'] != 'Success' + ): + self.log('Creating new media item failed: {}'.format(resp['newMediaItemResults'][0]['status'])) + return None + + return resp['newMediaItemResults'][0] diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index 2e8e103a..7d90ffda 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -10,7 +10,7 @@ from sys import exc_info from traceback import format_exc -from elodie.config import load_plugin_config +from elodie.config import load_config_for_plugin, load_plugin_config from elodie import log class ElodiePluginError(Exception): @@ -21,7 +21,13 @@ class PluginBase(object): __name__ = 'PluginBase' - def before(self, file_path, destination_path, media): + def __init__(self): + self.config_for_plugin = load_config_for_plugin(self.__name__) + + def after(self, file_path, destination_folder, final_file_path, media): + pass + + def before(self, file_path, destination_folder, media): pass def log(self, msg): diff --git a/elodie/plugins/runtimeerror/runtimeerror.py b/elodie/plugins/runtimeerror/runtimeerror.py index d4a486a4..a9d0bfdd 100644 --- a/elodie/plugins/runtimeerror/runtimeerror.py +++ b/elodie/plugins/runtimeerror/runtimeerror.py @@ -15,6 +15,6 @@ class RuntimeError(PluginBase): def __init__(self): pass - def before(self, file_path, destination_path, media): + def before(self, file_path, destination_folder, media): print(does_not_exist) diff --git a/elodie/plugins/throwerror/throwerror.py b/elodie/plugins/throwerror/throwerror.py index 0bd74a0d..78445dbb 100644 --- a/elodie/plugins/throwerror/throwerror.py +++ b/elodie/plugins/throwerror/throwerror.py @@ -15,5 +15,5 @@ class ThrowError(PluginBase): def __init__(self): pass - def before(self, file_path, destination_path, media): + def before(self, file_path, destination_folder, media): raise ElodiePluginError('Sample plugin error') diff --git a/elodie/tests/files/plugins/googlephotos/auth_file.json b/elodie/tests/files/plugins/googlephotos/auth_file.json new file mode 100644 index 00000000..d2fd25b1 --- /dev/null +++ b/elodie/tests/files/plugins/googlephotos/auth_file.json @@ -0,0 +1 @@ +{"scopes": ["https://www.googleapis.com/auth/photoslibrary", "https://www.googleapis.com/auth/photoslibrary.appendonly", "https://www.googleapis.com/auth/photoslibrary.sharing"], "id_token": null, "token": "ya29.Gls1B3ymBdURb3tBLAUJgxQtfTzKry0eaUqplbkHxfIYJH9sWvkalLwXprfAW-Ku0Fz8aP3jz7NrncJ2h58idSEgrYXPQ14iVSkwgUGE2gnsxZM6w4TbHz8ny8Yf", "client_id": "1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com", "token_uri": "https://oauth2.googleapis.com/token", "client_secret": "p84GOD_i7_PwDGbAmpnbwKiH", "refresh_token": "1/iHiK9Vbq9i5ebXScyQPDZf9_GJrDOWMvu-2zG9wRsDA"} \ No newline at end of file diff --git a/elodie/tests/files/plugins/googlephotos/secrets_file.json b/elodie/tests/files/plugins/googlephotos/secrets_file.json new file mode 100644 index 00000000..9e5170e0 --- /dev/null +++ b/elodie/tests/files/plugins/googlephotos/secrets_file.json @@ -0,0 +1 @@ +{"installed":{"client_id":"1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com","project_id":"elodietestphotos-1561522235041","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"p84GOD_i7_PwDGbAmpnbwKiH","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file diff --git a/elodie/tests/plugins/googlephotos/__init__.py b/elodie/tests/plugins/googlephotos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/elodie/tests/plugins/googlephotos/googlephotos_test.py b/elodie/tests/plugins/googlephotos/googlephotos_test.py new file mode 100644 index 00000000..78076ac0 --- /dev/null +++ b/elodie/tests/plugins/googlephotos/googlephotos_test.py @@ -0,0 +1,79 @@ +from __future__ import absolute_import +# Project imports +import mock +import os +import sys +from tempfile import gettempdir + +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) + +import helper +from elodie.config import load_config +from elodie.plugins.googlephotos.googlephotos import GooglePhotos + +# Globals to simplify mocking configs +auth_file = helper.get_file('plugins/googlephotos/auth_file.json') +secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json') +config_string = """ +[Plugins] +plugins=GooglePhotos + +[PluginGooglePhotos] +auth_file={} +secrets_file={} + """ +config_string_fmt = config_string.format( + auth_file, + secrets_file +) + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-set-session' % gettempdir()) +def test_googlephotos_set_session(): + with open('%s/config.ini-googlephotos-set-session' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + assert gp.session is None, gp.session + gp.set_session() + assert gp.session is not None, gp.session + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload' % gettempdir()) +def test_googlephotos_upload(): + with open('%s/config.ini-googlephotos-upload' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + gp.set_session() + status = gp.upload(helper.get_file('plain.jpg')) + + assert status is not None, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload-invalid-empty' % gettempdir()) +def test_googlephotos_upload_invalid_empty(): + with open('%s/config.ini-googlephotos-upload-invalid-empty' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + gp.set_session() + status = gp.upload(helper.get_file('invalid.jpg')) + + assert status is None, status From 0dffa733f0e26f0f17ec94b189ce6bbd326b6a89 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Mon, 8 Jul 2019 23:43:48 -0700 Subject: [PATCH 08/16] Add run_all_after() --- elodie/plugins/googlephotos/googlephotos.py | 1 + elodie/plugins/plugins.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index d059bbd6..74f31fd6 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -1,5 +1,6 @@ """ Google Photos plugin object. +Upload code adapted from https://github.com/eshmu/gphotos-upload .. moduleauthor:: Jaisen Mathai """ diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index 7d90ffda..fcc10231 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -70,8 +70,7 @@ def load(self): self.loaded = True - - def run_all_before(self, file_path, destination_path, media): + def run_all_before(self, file_path, destination_folder, media): self.load() """Process `before` methods of each plugin that was loaded. """ @@ -84,7 +83,7 @@ def run_all_before(self, file_path, destination_path, media): # If any other error occurs we log the message and proceed as usual. # By default, plugins don't change behavior. try: - this_method(file_path, destination_path, media) + this_method(file_path, destination_folder, media) except ElodiePluginError as err: log.warn('Plugin {} raised an exception: {}'.format(cls, err)) log.error(format_exc()) @@ -92,3 +91,6 @@ def run_all_before(self, file_path, destination_path, media): except: log.error(format_exc()) return pass_status + + def run_all_after(self, file_path, destination_folder, final_file_path, media): + pass From f7c822b8c28773a4f90aa5c4b2bdaf0faa65e4e1 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Wed, 10 Jul 2019 00:02:07 -0700 Subject: [PATCH 09/16] Add batch functions and commands --- elodie.py | 7 ++ elodie/plugins/googlephotos/googlephotos.py | 18 ++- elodie/plugins/plugins.py | 104 +++++++++++++++++- elodie/tests/elodie_test.py | 76 ++++++------- .../plugins/googlephotos/googlephotos_test.py | 29 +++++ elodie/tests/plugins_test.py | 45 +++++++- 6 files changed, 238 insertions(+), 41 deletions(-) diff --git a/elodie.py b/elodie.py index b8310a61..ccbc5783 100755 --- a/elodie.py +++ b/elodie.py @@ -27,6 +27,7 @@ from elodie.media.audio import Audio from elodie.media.photo import Photo from elodie.media.video import Video +from elodie.plugins.plugins import Plugins from elodie.result import Result @@ -70,6 +71,11 @@ def import_file(_file, destination, album_from_folder, trash, allow_duplicates): return dest_path or None +@click.command('batch') +def _batch(): + plugins = Plugins() + plugins.run_batch() + @click.command('import') @click.option('--destination', type=click.Path(file_okay=False), @@ -340,6 +346,7 @@ def main(): main.add_command(_update) main.add_command(_generate_db) main.add_command(_verify) +main.add_command(_batch) if __name__ == '__main__': diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 74f31fd6..1e237be1 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -49,7 +49,23 @@ def __init__(self): self.session = None def after(self, file_path, destination_folder, final_file_path, media): - pass + self.db.set(final_file_path, media) + + def batch(self): + queue = self.db.get_all() + status = True + count = 0 + for key in queue: + this_status = self.upload(key) + if(this_status): + # Remove from queue if successful then increment count + self.db.delete(key) + count = count + 1 + self.display('{} uploaded successfully.'.format(key)) + else: + status = False + self.display('{} failed to upload.'.format(key)) + return (status, count) def before(self, file_path, destination_folder, media): pass diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index fcc10231..fd53600d 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -6,11 +6,15 @@ from __future__ import print_function from builtins import object +from json import dumps, loads from importlib import import_module +from os.path import dirname, dirname, isdir, isfile +from os import mkdir from sys import exc_info from traceback import format_exc from elodie.config import load_config_for_plugin, load_plugin_config +from elodie.constants import application_directory from elodie import log class ElodiePluginError(Exception): @@ -23,16 +27,73 @@ class PluginBase(object): def __init__(self): self.config_for_plugin = load_config_for_plugin(self.__name__) + self.db = PluginDb(self.__name__) def after(self, file_path, destination_folder, final_file_path, media): pass + def batch(self): + pass + def before(self, file_path, destination_folder, media): pass def log(self, msg): log.info(msg) + def display(self, msg): + log.all(dumps( + {self.__name__: msg} + )) + +class PluginDb(object): + + def __init__(self, plugin_name): + self.db_file = '{}/plugins/{}.json'.format( + application_directory, + plugin_name.lower() + ) + + # If the plugin db directory does not exist, create it + if(not isdir(dirname(self.db_file))): + mkdir(dirname(self.db_file)) + + # If the db file does not exist we initialize it + if(not isfile(self.db_file)): + with open(self.db_file, 'w+') as f: + f.write(dumps({})) + + + def get(self, key): + with open(self.db_file, 'r') as f: + db = loads(f.read()) + + if(key not in db): + return None + + return db[key] + + def set(self, key, value): + with open(self.db_file, 'r') as f: + db = loads(f.read()) + + db[key] = value + with open(self.db_file, 'w') as f: + f.write(dumps(db)) + + def get_all(self): + with open(self.db_file, 'r') as f: + db = loads(f.read()) + return db + + def delete(self, key): + with open(self.db_file, 'r') as f: + db = loads(f.read()) + + # delete key without throwing an exception + db.pop(key, None) + with open(self.db_file, 'w') as f: + f.write(dumps(db)) class Plugins(object): @@ -71,9 +132,9 @@ def load(self): self.loaded = True def run_all_before(self, file_path, destination_folder, media): - self.load() """Process `before` methods of each plugin that was loaded. """ + self.load() pass_status = True for cls in self.classes: this_method = getattr(self.classes[cls], 'before') @@ -93,4 +154,43 @@ def run_all_before(self, file_path, destination_folder, media): return pass_status def run_all_after(self, file_path, destination_folder, final_file_path, media): - pass + """Process `before` methods of each plugin that was loaded. + """ + self.load() + pass_status = True + for cls in self.classes: + this_method = getattr(self.classes[cls], 'before') + # We try to call the plugin's `before()` method. + # If the method explicitly raises an ElodiePluginError we'll fail the import + # by setting pass_status to False. + # If any other error occurs we log the message and proceed as usual. + # By default, plugins don't change behavior. + try: + this_method(file_path, destination_folder, media) + except ElodiePluginError as err: + log.warn('Plugin {} raised an exception: {}'.format(cls, err)) + log.error(format_exc()) + pass_status = False + except: + log.error(format_exc()) + return pass_status + + def run_batch(self): + self.load() + pass_status = True + for cls in self.classes: + this_method = getattr(self.classes[cls], 'batch') + # We try to call the plugin's `before()` method. + # If the method explicitly raises an ElodiePluginError we'll fail the import + # by setting pass_status to False. + # If any other error occurs we log the message and proceed as usual. + # By default, plugins don't change behavior. + try: + this_method() + except ElodiePluginError as err: + log.warn('Plugin {} raised an exception: {}'.format(cls, err)) + log.error(format_exc()) + pass_status = False + except: + log.error(format_exc()) + return pass_status diff --git a/elodie/tests/elodie_test.py b/elodie/tests/elodie_test.py index ddbdf0b9..0ea387b3 100644 --- a/elodie/tests/elodie_test.py +++ b/elodie/tests/elodie_test.py @@ -23,6 +23,8 @@ from elodie.media.photo import Photo from elodie.media.text import Text from elodie.media.video import Video +from elodie.plugins.plugins import Plugins +from elodie.plugins.googlephotos.googlephotos import GooglePhotos os.environ['TZ'] = 'GMT' @@ -605,43 +607,43 @@ def test_verify_error(): assert origin in result.output, result.output assert 'Error 1' in result.output, result.output -# @mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir()) -# def test_cli_batch_plugin_googlephotos(): -# auth_file = helper.get_file('plugins/googlephotos/auth_file.json') -# secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json') -# config_string = """ -# [Plugins] -# plugins=GooglePhotos - -# [PluginGooglePhotos] -# auth_file={} -# secrets_file={} -# """ -# config_string_fmt = config_string.format( -# auth_file, -# secrets_file -# ) -# with open('%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir(), 'w') as f: -# f.write(config_string_fmt) - -# if hasattr(load_config, 'config'): -# del load_config.config - -# final_file_path_1 = helper.get_file('plain.jpg') -# final_file_path_2 = helper.get_file('no-exif.jpg') -# sample_metadata_1 = Photo(final_file_path_1).get_metadata() -# sample_metadata_2 = Photo(final_file_path_2).get_metadata() -# gp = GooglePhotos() -# gp.after('', '', final_file_path_1, sample_metadata_1) -# gp.after('', '', final_file_path_2, sample_metadata_1) - -# if hasattr(load_config, 'config'): -# del load_config.config - -# runner = CliRunner() -# result = runner.invoke(elodie._batch) -# assert "elodie/elodie/tests/files/plain.jpg uploaded successfully.\"}\n" in result.output, result.output -# assert "elodie/elodie/tests/files/no-exif.jpg uploaded successfully.\"}\n" in result.output, result.output +@mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir()) +def test_cli_batch_plugin_googlephotos(): + auth_file = helper.get_file('plugins/googlephotos/auth_file.json') + secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json') + config_string = """ + [Plugins] + plugins=GooglePhotos + + [PluginGooglePhotos] + auth_file={} + secrets_file={} + """ + config_string_fmt = config_string.format( + auth_file, + secrets_file + ) + with open('%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path_1 = helper.get_file('plain.jpg') + final_file_path_2 = helper.get_file('no-exif.jpg') + sample_metadata_1 = Photo(final_file_path_1).get_metadata() + sample_metadata_2 = Photo(final_file_path_2).get_metadata() + gp = GooglePhotos() + gp.after('', '', final_file_path_1, sample_metadata_1) + gp.after('', '', final_file_path_2, sample_metadata_1) + + if hasattr(load_config, 'config'): + del load_config.config + + runner = CliRunner() + result = runner.invoke(elodie._batch) + assert "elodie/elodie/tests/files/plain.jpg uploaded successfully.\"}\n" in result.output, result.output + assert "elodie/elodie/tests/files/no-exif.jpg uploaded successfully.\"}\n" in result.output, result.output def test_cli_debug_import(): runner = CliRunner() diff --git a/elodie/tests/plugins/googlephotos/googlephotos_test.py b/elodie/tests/plugins/googlephotos/googlephotos_test.py index 78076ac0..71ed6753 100644 --- a/elodie/tests/plugins/googlephotos/googlephotos_test.py +++ b/elodie/tests/plugins/googlephotos/googlephotos_test.py @@ -77,3 +77,32 @@ def test_googlephotos_upload_invalid_empty(): status = gp.upload(helper.get_file('invalid.jpg')) assert status is None, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-batch' % gettempdir()) +def test_googlephotos_batch(): + with open('%s/config.ini-googlephotos-batch' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path = helper.get_file('plain.jpg') + gp = GooglePhotos() + gp.after('', '', final_file_path, {'foo': 'bar'}) + db_row = gp.db.get(final_file_path) + assert db_row == {'foo': 'bar'}, db_row + + status, count = gp.batch() + db_row_after = gp.db.get(final_file_path) + assert status == True, status + assert count == 1, count + assert db_row_after is None, db_row_after + + + if hasattr(load_config, 'config'): + del load_config.config + + + gp.set_session() + status = gp.upload(helper.get_file('invalid.jpg')) + + assert status is None, status diff --git a/elodie/tests/plugins_test.py b/elodie/tests/plugins_test.py index 15cfd708..afd71ca2 100644 --- a/elodie/tests/plugins_test.py +++ b/elodie/tests/plugins_test.py @@ -8,8 +8,9 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) from . import helper +from .helper import test_setup_func from elodie.config import load_config -from elodie.plugins.plugins import Plugins +from elodie.plugins.plugins import Plugins, PluginBase, PluginDb @mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-unset-backwards-compat' % gettempdir()) def test_load_plugins_unset_backwards_compat(): @@ -199,3 +200,45 @@ def test_throw_error_runtime_error(): del load_config.config assert status == True, status + +def test_plugin_base_inherits_db(): + plugin_base = PluginBase() + assert hasattr(plugin_base.db, 'get') + assert hasattr(plugin_base.db, 'set') + assert hasattr(plugin_base.db, 'get_all') + assert hasattr(plugin_base.db, 'delete') + +def test_db_initialize_file(): + db = PluginDb('foobar') + try: + os.remove(db.db_file) + except OSError: + pass + db = PluginDb('foobar') + +def test_db_get_then_set_then_get_then_delete(): + db = PluginDb('foobar') + foo = db.get('foo') + assert foo is None, foo + db.set('foo', 'bar') + foo = db.get('foo') + assert foo == 'bar', foo + db.delete('foo') + foo = db.get('foo') + assert foo is None, foo + +def test_db_get_all(): + # we initialize the db to get the file path to delete then reinitialize + db = PluginDb('foobar') + try: + os.remove(db.db_file) + except OSError: + pass + db = PluginDb('foobar') + db.set('a', '1') + db.set('b', '2') + db.set('c', '3') + db.set('d', '4') + all_rows = db.get_all() + + assert all_rows == {'a': '1', 'b': '2', 'c': '3', 'd': '4'}, all_rows From 7325bedc1667ba469f39540170da1d48683e8f40 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Wed, 10 Jul 2019 00:09:00 -0700 Subject: [PATCH 10/16] Include google photos requirements.txt in .travis.yml --- .travis.yml | 1 + .../plugins/googlephotos/requirements.txt | 0 elodie/tests/plugins_test.py | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) rename requirements-google.txt => elodie/plugins/googlephotos/requirements.txt (100%) diff --git a/.travis.yml b/.travis.yml index 894bfa49..e624f4e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ before_install: - "sudo apt-get install python-dev python-pip -y" install: - "pip install -r elodie/tests/requirements.txt" + - "pip install -r elodie/plugins/googlephotos/requirements.txt" - "pip install coveralls" before_script: - "mkdir ~/.elodie" diff --git a/requirements-google.txt b/elodie/plugins/googlephotos/requirements.txt similarity index 100% rename from requirements-google.txt rename to elodie/plugins/googlephotos/requirements.txt diff --git a/elodie/tests/plugins_test.py b/elodie/tests/plugins_test.py index afd71ca2..f97053ed 100644 --- a/elodie/tests/plugins_test.py +++ b/elodie/tests/plugins_test.py @@ -8,7 +8,6 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) from . import helper -from .helper import test_setup_func from elodie.config import load_config from elodie.plugins.plugins import Plugins, PluginBase, PluginDb From aa58d7f98b12754057c2fd5188fce35730adaa06 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Wed, 10 Jul 2019 02:04:26 -0700 Subject: [PATCH 11/16] Changes from debuging real usage --- elodie.py | 5 ++- elodie/filesystem.py | 4 +-- elodie/plugins/dummy/dummy.py | 2 +- elodie/plugins/googlephotos/googlephotos.py | 14 +++++--- elodie/plugins/plugins.py | 39 ++++++++++++--------- elodie/plugins/runtimeerror/runtimeerror.py | 2 +- elodie/plugins/throwerror/throwerror.py | 2 +- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/elodie.py b/elodie.py index ccbc5783..e9b629a3 100755 --- a/elodie.py +++ b/elodie.py @@ -72,7 +72,10 @@ def import_file(_file, destination, album_from_folder, trash, allow_duplicates): return dest_path or None @click.command('batch') -def _batch(): +@click.option('--debug', default=False, is_flag=True, + help='Override the value in constants.py with True.') +def _batch(debug): + constants.debug = debug plugins = Plugins() plugins.run_batch() diff --git a/elodie/filesystem.py b/elodie/filesystem.py index 14731741..8a71ab5f 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -532,7 +532,7 @@ def process_file(self, _file, destination, media, **kwargs): # Run `before()` for every loaded plugin and if any of them raise an exception # then we skip importing the file and log a message. - plugins_run_before_status = self.plugins.run_all_before(_file, destination, media) + plugins_run_before_status = self.plugins.run_all_before(_file, destination) if(plugins_run_before_status == False): log.warn('At least one plugin pre-run failed for %s' % _file) return @@ -596,7 +596,7 @@ def process_file(self, _file, destination, media, **kwargs): # Run `after()` for every loaded plugin and if any of them raise an exception # then we skip importing the file and log a message. - plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, media) + plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, metadata) if(plugins_run_after_status == False): log.warn('At least one plugin pre-run failed for %s' % _file) return diff --git a/elodie/plugins/dummy/dummy.py b/elodie/plugins/dummy/dummy.py index 2ea66f62..4dd220f9 100644 --- a/elodie/plugins/dummy/dummy.py +++ b/elodie/plugins/dummy/dummy.py @@ -15,6 +15,6 @@ class Dummy(PluginBase): def __init__(self): self.before_ran = False - def before(self, file_path, destination_folder, media): + def before(self, file_path, destination_folder): self.before_ran = True diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 1e237be1..2ce5c89d 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -48,8 +48,9 @@ def __init__(self): self.auth_file = self.config_for_plugin['auth_file'] self.session = None - def after(self, file_path, destination_folder, final_file_path, media): - self.db.set(final_file_path, media) + def after(self, file_path, destination_folder, final_file_path, metadata): + self.log(u'Added {} to db.'.format(final_file_path)) + self.db.set(final_file_path, metadata['original_name']) def batch(self): queue = self.db.get_all() @@ -67,7 +68,7 @@ def batch(self): self.display('{} failed to upload.'.format(key)) return (status, count) - def before(self, file_path, destination_folder, media): + def before(self, file_path, destination_folder): pass def set_session(self): @@ -77,7 +78,6 @@ def set_session(self): try: creds = Credentials.from_authorized_user_file(self.auth_file, self.scopes) except: - print(self.secrets_file) flow = InstalledAppFlow.from_client_secrets_file(self.secrets_file, self.scopes) creds = flow.run_local_server() cred_dict = { @@ -123,7 +123,11 @@ def upload(self, path_to_photo): 'newMediaItemResults' not in resp or 'status' not in resp['newMediaItemResults'][0] or 'message' not in resp['newMediaItemResults'][0]['status'] or - resp['newMediaItemResults'][0]['status']['message'] != 'Success' + ( + resp['newMediaItemResults'][0]['status']['message'] != 'Success' and # photos + resp['newMediaItemResults'][0]['status']['message'] != 'OK' # videos + ) + ): self.log('Creating new media item failed: {}'.format(resp['newMediaItemResults'][0]['status'])) return None diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index fd53600d..1f4dc564 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -6,6 +6,8 @@ from __future__ import print_function from builtins import object +import io + from json import dumps, loads from importlib import import_module from os.path import dirname, dirname, isdir, isfile @@ -29,17 +31,19 @@ def __init__(self): self.config_for_plugin = load_config_for_plugin(self.__name__) self.db = PluginDb(self.__name__) - def after(self, file_path, destination_folder, final_file_path, media): + def after(self, file_path, destination_folder, final_file_path, metadata): pass def batch(self): pass - def before(self, file_path, destination_folder, media): + def before(self, file_path, destination_folder): pass def log(self, msg): - log.info(msg) + log.info(dumps( + {self.__name__: msg} + )) def display(self, msg): log.all(dumps( @@ -60,12 +64,12 @@ def __init__(self, plugin_name): # If the db file does not exist we initialize it if(not isfile(self.db_file)): - with open(self.db_file, 'w+') as f: + with io.open(self.db_file, 'w+') as f: f.write(dumps({})) def get(self, key): - with open(self.db_file, 'r') as f: + with io.open(self.db_file, 'r') as f: db = loads(f.read()) if(key not in db): @@ -74,26 +78,26 @@ def get(self, key): return db[key] def set(self, key, value): - with open(self.db_file, 'r') as f: + with io.open(self.db_file, 'r') as f: db = loads(f.read()) db[key] = value - with open(self.db_file, 'w') as f: - f.write(dumps(db)) + with io.open(self.db_file, 'rb+') as f: + f.write(unicode(dumps(db, ensure_ascii=False).encode('utf8'))) def get_all(self): - with open(self.db_file, 'r') as f: + with io.open(self.db_file, 'r') as f: db = loads(f.read()) return db def delete(self, key): - with open(self.db_file, 'r') as f: + with io.open(self.db_file, 'r') as f: db = loads(f.read()) # delete key without throwing an exception db.pop(key, None) - with open(self.db_file, 'w') as f: - f.write(dumps(db)) + with io.open(self.db_file, 'rb+') as f: + f.write(unicode(dumps(db, ensure_ascii=False).encode('utf8'))) class Plugins(object): @@ -131,7 +135,7 @@ def load(self): self.loaded = True - def run_all_before(self, file_path, destination_folder, media): + def run_all_before(self, file_path, destination_folder): """Process `before` methods of each plugin that was loaded. """ self.load() @@ -144,7 +148,7 @@ def run_all_before(self, file_path, destination_folder, media): # If any other error occurs we log the message and proceed as usual. # By default, plugins don't change behavior. try: - this_method(file_path, destination_folder, media) + this_method(file_path, destination_folder) except ElodiePluginError as err: log.warn('Plugin {} raised an exception: {}'.format(cls, err)) log.error(format_exc()) @@ -153,20 +157,21 @@ def run_all_before(self, file_path, destination_folder, media): log.error(format_exc()) return pass_status - def run_all_after(self, file_path, destination_folder, final_file_path, media): + def run_all_after(self, file_path, destination_folder, final_file_path, metadata): """Process `before` methods of each plugin that was loaded. """ self.load() pass_status = True for cls in self.classes: - this_method = getattr(self.classes[cls], 'before') + this_method = getattr(self.classes[cls], 'after') # We try to call the plugin's `before()` method. # If the method explicitly raises an ElodiePluginError we'll fail the import # by setting pass_status to False. # If any other error occurs we log the message and proceed as usual. # By default, plugins don't change behavior. try: - this_method(file_path, destination_folder, media) + this_method(file_path, destination_folder, final_file_path, metadata) + log.info('Called after() for {}'.format(cls)) except ElodiePluginError as err: log.warn('Plugin {} raised an exception: {}'.format(cls, err)) log.error(format_exc()) diff --git a/elodie/plugins/runtimeerror/runtimeerror.py b/elodie/plugins/runtimeerror/runtimeerror.py index a9d0bfdd..2bbbfeb3 100644 --- a/elodie/plugins/runtimeerror/runtimeerror.py +++ b/elodie/plugins/runtimeerror/runtimeerror.py @@ -15,6 +15,6 @@ class RuntimeError(PluginBase): def __init__(self): pass - def before(self, file_path, destination_folder, media): + def before(self, file_path, destination_folder): print(does_not_exist) diff --git a/elodie/plugins/throwerror/throwerror.py b/elodie/plugins/throwerror/throwerror.py index 78445dbb..5c8987d2 100644 --- a/elodie/plugins/throwerror/throwerror.py +++ b/elodie/plugins/throwerror/throwerror.py @@ -15,5 +15,5 @@ class ThrowError(PluginBase): def __init__(self): pass - def before(self, file_path, destination_folder, media): + def before(self, file_path, destination_folder): raise ElodiePluginError('Sample plugin error') From 15b7f1615e34544b9994ac22a55975c5b918135c Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Wed, 10 Jul 2019 02:10:59 -0700 Subject: [PATCH 12/16] fix test errors --- elodie/plugins/plugins.py | 2 +- elodie/tests/plugins_test.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index 1f4dc564..cd619823 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -65,7 +65,7 @@ def __init__(self, plugin_name): # If the db file does not exist we initialize it if(not isfile(self.db_file)): with io.open(self.db_file, 'w+') as f: - f.write(dumps({})) + f.write(unicode(dumps({}))) def get(self, key): diff --git a/elodie/tests/plugins_test.py b/elodie/tests/plugins_test.py index f97053ed..60155fb6 100644 --- a/elodie/tests/plugins_test.py +++ b/elodie/tests/plugins_test.py @@ -134,7 +134,7 @@ def test_run_before(): plugins = Plugins() plugins.load() before_ran_1 = plugins.classes['Dummy'].before_ran - plugins.run_all_before('', '', '') + plugins.run_all_before('', '') before_ran_2 = plugins.classes['Dummy'].before_ran if hasattr(load_config, 'config'): @@ -155,7 +155,7 @@ def test_throw_error(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '', '') + status = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config @@ -174,7 +174,7 @@ def test_throw_error_one_of_many(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '', '') + status = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config @@ -193,7 +193,7 @@ def test_throw_error_runtime_error(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '', '') + status = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config From 6ad77f8c9a471947596a0da6201c3186f9f56be3 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Thu, 11 Jul 2019 00:47:02 -0700 Subject: [PATCH 13/16] More changes --- elodie/plugins/googlephotos/googlephotos.py | 10 +++- elodie/plugins/plugins.py | 15 +++--- .../plugins/googlephotos/googlephotos_test.py | 47 ++++++++++++++++++- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py index 2ce5c89d..0f52fe1a 100644 --- a/elodie/plugins/googlephotos/googlephotos.py +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -14,6 +14,8 @@ from google.auth.transport.requests import AuthorizedSession from google.oauth2.credentials import Credentials +from elodie.media.photo import Photo +from elodie.media.video import Video from elodie.plugins.plugins import PluginBase class GooglePhotos(PluginBase): @@ -49,8 +51,12 @@ def __init__(self): self.session = None def after(self, file_path, destination_folder, final_file_path, metadata): - self.log(u'Added {} to db.'.format(final_file_path)) - self.db.set(final_file_path, metadata['original_name']) + extension = metadata['extension'] + if(extension in Photo.extensions or extension in Video.extensions): + self.log(u'Added {} to db.'.format(final_file_path)) + self.db.set(final_file_path, metadata['original_name']) + else: + self.log(u'Skipping {} which is not a supported media type.'.format(final_file_path)) def batch(self): queue = self.db.get_all() diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index cd619823..378c0ca4 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -64,7 +64,7 @@ def __init__(self, plugin_name): # If the db file does not exist we initialize it if(not isfile(self.db_file)): - with io.open(self.db_file, 'w+') as f: + with io.open(self.db_file, 'wb') as f: f.write(unicode(dumps({}))) @@ -79,11 +79,13 @@ def get(self, key): def set(self, key, value): with io.open(self.db_file, 'r') as f: - db = loads(f.read()) + data = f.read() + db = loads(data) db[key] = value - with io.open(self.db_file, 'rb+') as f: - f.write(unicode(dumps(db, ensure_ascii=False).encode('utf8'))) + new_content = unicode(dumps(db, ensure_ascii=False).encode('utf8')) + with io.open(self.db_file, 'wb') as f: + f.write(new_content) def get_all(self): with io.open(self.db_file, 'r') as f: @@ -96,8 +98,9 @@ def delete(self, key): # delete key without throwing an exception db.pop(key, None) - with io.open(self.db_file, 'rb+') as f: - f.write(unicode(dumps(db, ensure_ascii=False).encode('utf8'))) + new_content = unicode(dumps(db, ensure_ascii=False).encode('utf8')) + with io.open(self.db_file, 'wb') as f: + f.write(new_content) class Plugins(object): diff --git a/elodie/tests/plugins/googlephotos/googlephotos_test.py b/elodie/tests/plugins/googlephotos/googlephotos_test.py index 71ed6753..2f2bc481 100644 --- a/elodie/tests/plugins/googlephotos/googlephotos_test.py +++ b/elodie/tests/plugins/googlephotos/googlephotos_test.py @@ -11,6 +11,8 @@ import helper from elodie.config import load_config from elodie.plugins.googlephotos.googlephotos import GooglePhotos +from elodie.media.audio import Audio +from elodie.media.photo import Photo # Globals to simplify mocking configs auth_file = helper.get_file('plugins/googlephotos/auth_file.json') @@ -28,6 +30,10 @@ secrets_file ) +sample_photo = Photo(helper.get_file('plain.jpg')) +sample_metadata = sample_photo.get_metadata() +sample_metadata['original_name'] = 'foobar' + @mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-set-session' % gettempdir()) def test_googlephotos_set_session(): with open('%s/config.ini-googlephotos-set-session' % gettempdir(), 'w') as f: @@ -44,6 +50,43 @@ def test_googlephotos_set_session(): gp.set_session() assert gp.session is not None, gp.session +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-after-supported' % gettempdir()) +def test_googlephotos_after_supported(): + with open('%s/config.ini-googlephotos-after-supported' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path = helper.get_file('plain.jpg') + gp = GooglePhotos() + gp.after('', '', final_file_path, sample_metadata) + db_row = gp.db.get(final_file_path) + + if hasattr(load_config, 'config'): + del load_config.config + + assert db_row == 'foobar', db_row + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-after-unsupported' % gettempdir()) +def test_googlephotos_after_unsupported(): + with open('%s/config.ini-googlephotos-after-unsupported' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path = helper.get_file('audio.m4a') + sample_photo = Audio(final_file_path) + sample_metadata = sample_photo.get_metadata() + sample_metadata['original_name'] = 'foobar' + gp = GooglePhotos() + gp.after('', '', final_file_path, sample_metadata) + db_row = gp.db.get(final_file_path) + + if hasattr(load_config, 'config'): + del load_config.config + + assert db_row == None, db_row + @mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload' % gettempdir()) def test_googlephotos_upload(): with open('%s/config.ini-googlephotos-upload' % gettempdir(), 'w') as f: @@ -87,9 +130,9 @@ def test_googlephotos_batch(): final_file_path = helper.get_file('plain.jpg') gp = GooglePhotos() - gp.after('', '', final_file_path, {'foo': 'bar'}) + gp.after('', '', final_file_path, sample_metadata) db_row = gp.db.get(final_file_path) - assert db_row == {'foo': 'bar'}, db_row + assert db_row == 'foobar', db_row status, count = gp.batch() db_row_after = gp.db.get(final_file_path) From 68d3e62e3279d3d15bfd4dfa8b96fab84736a9ea Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Thu, 11 Jul 2019 22:01:23 -0700 Subject: [PATCH 14/16] fix plugins by encoding json to utf8 --- elodie/compatability.py | 5 +++++ elodie/plugins/plugins.py | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/elodie/compatability.py b/elodie/compatability.py index d4a68ddf..bc6f5f3a 100644 --- a/elodie/compatability.py +++ b/elodie/compatability.py @@ -24,6 +24,11 @@ def _decode(string, encoding=sys.getfilesystemencoding()): return string +def _bytes(string): + if constants.python_version == 3: + return bytes(string, 'utf8') + else: + return bytes(string) def _copyfile(src, dst): # shutil.copy seems slow, changing to streaming according to diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index 378c0ca4..f4848412 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -15,6 +15,7 @@ from sys import exc_info from traceback import format_exc +from elodie.compatability import _bytes from elodie.config import load_config_for_plugin, load_plugin_config from elodie.constants import application_directory from elodie import log @@ -65,7 +66,7 @@ def __init__(self, plugin_name): # If the db file does not exist we initialize it if(not isfile(self.db_file)): with io.open(self.db_file, 'wb') as f: - f.write(unicode(dumps({}))) + f.write(_bytes(dumps({}))) def get(self, key): @@ -83,7 +84,7 @@ def set(self, key, value): db = loads(data) db[key] = value - new_content = unicode(dumps(db, ensure_ascii=False).encode('utf8')) + new_content = dumps(db, ensure_ascii=False).encode('utf8') with io.open(self.db_file, 'wb') as f: f.write(new_content) @@ -98,7 +99,7 @@ def delete(self, key): # delete key without throwing an exception db.pop(key, None) - new_content = unicode(dumps(db, ensure_ascii=False).encode('utf8')) + new_content = dumps(db, ensure_ascii=False).encode('utf8') with io.open(self.db_file, 'wb') as f: f.write(new_content) From b9c99a9f46f04e20fdeb7772a60964930cbc5b8f Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Thu, 11 Jul 2019 22:10:20 -0700 Subject: [PATCH 15/16] increase coverage for plugins test --- elodie/tests/plugins_test.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/elodie/tests/plugins_test.py b/elodie/tests/plugins_test.py index 60155fb6..7ee98749 100644 --- a/elodie/tests/plugins_test.py +++ b/elodie/tests/plugins_test.py @@ -155,12 +155,14 @@ def test_throw_error(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '') + status_before = plugins.run_all_before('', '') + status_after = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status == False, status + assert status_before == False, status_before + assert status_after == False, status_after @mock.patch('elodie.config.config_file', '%s/config.ini-throw-error-one-of-many' % gettempdir()) def test_throw_error_one_of_many(): @@ -174,12 +176,14 @@ def test_throw_error_one_of_many(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '') + status_before = plugins.run_all_before('', '') + status_after = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status == False, status + assert status_before == False, status_before + assert status_after == False, status_after @mock.patch('elodie.config.config_file', '%s/config.ini-throw-runtime-error' % gettempdir()) def test_throw_error_runtime_error(): @@ -193,12 +197,14 @@ def test_throw_error_runtime_error(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '') + status_before = plugins.run_all_before('', '') + status_after = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status == True, status + assert status_before == True, status_before + assert status_after == True, status_after def test_plugin_base_inherits_db(): plugin_base = PluginBase() From fbde369d69bf131c04e30d452df4f360e2572ecc Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Thu, 11 Jul 2019 22:25:28 -0700 Subject: [PATCH 16/16] increase coverage for plugin tests --- elodie/plugins/plugins.py | 33 ++++++++++++++----------- elodie/plugins/throwerror/throwerror.py | 8 +++++- elodie/tests/plugins_test.py | 18 +++++++++----- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index f4848412..4e62cdb1 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -139,65 +139,68 @@ def load(self): self.loaded = True - def run_all_before(self, file_path, destination_folder): + def run_all_after(self, file_path, destination_folder, final_file_path, metadata): """Process `before` methods of each plugin that was loaded. """ self.load() pass_status = True for cls in self.classes: - this_method = getattr(self.classes[cls], 'before') + this_method = getattr(self.classes[cls], 'after') # We try to call the plugin's `before()` method. # If the method explicitly raises an ElodiePluginError we'll fail the import # by setting pass_status to False. # If any other error occurs we log the message and proceed as usual. # By default, plugins don't change behavior. try: - this_method(file_path, destination_folder) + this_method(file_path, destination_folder, final_file_path, metadata) + log.info('Called after() for {}'.format(cls)) except ElodiePluginError as err: - log.warn('Plugin {} raised an exception: {}'.format(cls, err)) + log.warn('Plugin {} raised an exception in run_all_before: {}'.format(cls, err)) log.error(format_exc()) + log.error('false') pass_status = False except: log.error(format_exc()) return pass_status - def run_all_after(self, file_path, destination_folder, final_file_path, metadata): - """Process `before` methods of each plugin that was loaded. - """ + def run_batch(self): self.load() pass_status = True for cls in self.classes: - this_method = getattr(self.classes[cls], 'after') + this_method = getattr(self.classes[cls], 'batch') # We try to call the plugin's `before()` method. # If the method explicitly raises an ElodiePluginError we'll fail the import # by setting pass_status to False. # If any other error occurs we log the message and proceed as usual. # By default, plugins don't change behavior. try: - this_method(file_path, destination_folder, final_file_path, metadata) - log.info('Called after() for {}'.format(cls)) + this_method() + log.info('Called batch() for {}'.format(cls)) except ElodiePluginError as err: - log.warn('Plugin {} raised an exception: {}'.format(cls, err)) + log.warn('Plugin {} raised an exception in run_batch: {}'.format(cls, err)) log.error(format_exc()) pass_status = False except: log.error(format_exc()) return pass_status - def run_batch(self): + def run_all_before(self, file_path, destination_folder): + """Process `before` methods of each plugin that was loaded. + """ self.load() pass_status = True for cls in self.classes: - this_method = getattr(self.classes[cls], 'batch') + this_method = getattr(self.classes[cls], 'before') # We try to call the plugin's `before()` method. # If the method explicitly raises an ElodiePluginError we'll fail the import # by setting pass_status to False. # If any other error occurs we log the message and proceed as usual. # By default, plugins don't change behavior. try: - this_method() + this_method(file_path, destination_folder) + log.info('Called before() for {}'.format(cls)) except ElodiePluginError as err: - log.warn('Plugin {} raised an exception: {}'.format(cls, err)) + log.warn('Plugin {} raised an exception in run_all_after: {}'.format(cls, err)) log.error(format_exc()) pass_status = False except: diff --git a/elodie/plugins/throwerror/throwerror.py b/elodie/plugins/throwerror/throwerror.py index 5c8987d2..feb7ec64 100644 --- a/elodie/plugins/throwerror/throwerror.py +++ b/elodie/plugins/throwerror/throwerror.py @@ -15,5 +15,11 @@ class ThrowError(PluginBase): def __init__(self): pass + def after(self, file_path, destination_folder, final_file_path, metadata): + raise ElodiePluginError('Sample plugin error for after') + + def batch(self): + raise ElodiePluginError('Sample plugin error for batch') + def before(self, file_path, destination_folder): - raise ElodiePluginError('Sample plugin error') + raise ElodiePluginError('Sample plugin error for before') diff --git a/elodie/tests/plugins_test.py b/elodie/tests/plugins_test.py index 7ee98749..9f36010e 100644 --- a/elodie/tests/plugins_test.py +++ b/elodie/tests/plugins_test.py @@ -155,14 +155,16 @@ def test_throw_error(): plugins = Plugins() plugins.load() + status_after = plugins.run_all_after('', '', '', '') + status_batch = plugins.run_batch() status_before = plugins.run_all_before('', '') - status_after = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status_before == False, status_before assert status_after == False, status_after + assert status_batch == False, status_batch + assert status_before == False, status_before @mock.patch('elodie.config.config_file', '%s/config.ini-throw-error-one-of-many' % gettempdir()) def test_throw_error_one_of_many(): @@ -176,14 +178,16 @@ def test_throw_error_one_of_many(): plugins = Plugins() plugins.load() + status_after = plugins.run_all_after('', '', '', '') + status_batch = plugins.run_batch() status_before = plugins.run_all_before('', '') - status_after = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status_before == False, status_before assert status_after == False, status_after + assert status_batch == False, status_batch + assert status_before == False, status_before @mock.patch('elodie.config.config_file', '%s/config.ini-throw-runtime-error' % gettempdir()) def test_throw_error_runtime_error(): @@ -197,14 +201,16 @@ def test_throw_error_runtime_error(): plugins = Plugins() plugins.load() + status_after = plugins.run_all_after('', '', '', '') + status_batch = plugins.run_batch() status_before = plugins.run_all_before('', '') - status_after = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status_before == True, status_before assert status_after == True, status_after + assert status_batch == True, status_batch + assert status_before == True, status_before def test_plugin_base_inherits_db(): plugin_base = PluginBase()