From 3276ccfd26969a9e073226eed92df73ef3c31d02 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Fri, 12 Jul 2019 00:40:38 -0700 Subject: [PATCH] Add a plugin to upload photos to Google Photos (#319) Fixes #315. This PR aims to address the [recent changes](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/) in Google Photos + Google Drive where syncing between the two is no longer supported. It works by uploading photos as part of the import process to add a copy of every photo in your library to Google Photos. Google Drive is not required for this plugin to work. This plugin lets you have all your photos in Google Photos without relying on Google Drive. You can use another cloud storage service like iCloud or Dropbox or no cloud storage at all. - [x] Add tests for `after()` plugin methods. - [x] Add support for storage/async support. - [x] Include plugins into code coverage. - [x] Sweep code and clean up and add comments. --- .travis.yml | 1 + Readme.md | 4 +- elodie.py | 12 ++ elodie/compatability.py | 5 + elodie/config.py | 10 + elodie/filesystem.py | 10 +- elodie/plugins/dummy/dummy.py | 3 +- elodie/plugins/googlephotos/Readme.markdown | 63 ++++++ elodie/plugins/googlephotos/__init__.py | 0 elodie/plugins/googlephotos/googlephotos.py | 159 +++++++++++++++ .../plugins/googlephotos/requirements.txt | 0 elodie/plugins/plugins.py | 151 +++++++++++++- elodie/plugins/runtimeerror/runtimeerror.py | 8 +- elodie/plugins/throwerror/throwerror.py | 11 +- elodie/tests/elodie_test.py | 76 +++---- .../files/plugins/googlephotos/auth_file.json | 1 + .../plugins/googlephotos/secrets_file.json | 1 + elodie/tests/plugins/__init__.py | 0 elodie/tests/plugins/googlephotos/__init__.py | 0 .../plugins/googlephotos/googlephotos_test.py | 185 ++++++++++++++++++ elodie/tests/plugins_test.py | 78 ++++++-- 21 files changed, 711 insertions(+), 67 deletions(-) create mode 100644 elodie/plugins/googlephotos/Readme.markdown create mode 100644 elodie/plugins/googlephotos/__init__.py create mode 100644 elodie/plugins/googlephotos/googlephotos.py rename requirements-google.txt => elodie/plugins/googlephotos/requirements.txt (100%) create mode 100644 elodie/tests/files/plugins/googlephotos/auth_file.json create mode 100644 elodie/tests/files/plugins/googlephotos/secrets_file.json create mode 100644 elodie/tests/plugins/__init__.py create mode 100644 elodie/tests/plugins/googlephotos/__init__.py create mode 100644 elodie/tests/plugins/googlephotos/googlephotos_test.py diff --git a/.travis.yml b/.travis.yml index 894bfa49..e624f4e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ before_install: - "sudo apt-get install python-dev python-pip -y" install: - "pip install -r elodie/tests/requirements.txt" + - "pip install -r elodie/plugins/googlephotos/requirements.txt" - "pip install coveralls" before_script: - "mkdir ~/.elodie" diff --git a/Readme.md b/Readme.md index ae6083ab..3a1f630f 100644 --- a/Readme.md +++ b/Readme.md @@ -260,7 +260,7 @@ You can configure how Elodie names your files using placeholders. This works sim If you'd like to specify your own naming convention it's recommended you include something that's mostly unique like the time including seconds. You'll need to include a `[File]` section in your `config.ini` file with a name attribute. If a placeholder doesn't have a value then it plus any preceding characters which are not alphabetic are removed. -By default the resulting filename is all lowercased. To change this behavior to upppercasing add capitalization=upper. +By default the resulting filename is all lowercased. To change this behavior to uppercasing add capitalization=upper. ``` [File] @@ -270,7 +270,7 @@ name=%date-%original_name-%title.jpg date=%Y-%m-%b-%H-%M-%S name=%date-%original_name-%album.jpg -capitalization=uppper +capitalization=upper # -> 2012-05-MAR-12-59-30-DSC_1234-MY-ALBUM.JPG ``` diff --git a/elodie.py b/elodie.py index b8310a61..7d6e2caa 100755 --- a/elodie.py +++ b/elodie.py @@ -27,6 +27,7 @@ from elodie.media.audio import Audio from elodie.media.photo import Photo from elodie.media.video import Video +from elodie.plugins.plugins import Plugins from elodie.result import Result @@ -70,6 +71,16 @@ def import_file(_file, destination, album_from_folder, trash, allow_duplicates): return dest_path or None +@click.command('batch') +@click.option('--debug', default=False, is_flag=True, + help='Override the value in constants.py with True.') +def _batch(debug): + """Run batch() for all plugins. + """ + constants.debug = debug + plugins = Plugins() + plugins.run_batch() + @click.command('import') @click.option('--destination', type=click.Path(file_okay=False), @@ -340,6 +351,7 @@ def main(): main.add_command(_update) main.add_command(_generate_db) main.add_command(_verify) +main.add_command(_batch) if __name__ == '__main__': diff --git a/elodie/compatability.py b/elodie/compatability.py index d4a68ddf..bc6f5f3a 100644 --- a/elodie/compatability.py +++ b/elodie/compatability.py @@ -24,6 +24,11 @@ def _decode(string, encoding=sys.getfilesystemencoding()): return string +def _bytes(string): + if constants.python_version == 3: + return bytes(string, 'utf8') + else: + return bytes(string) def _copyfile(src, dst): # shutil.copy seems slow, changing to streaming according to diff --git a/elodie/config.py b/elodie/config.py index 4bfb6ef2..24e54be8 100644 --- a/elodie/config.py +++ b/elodie/config.py @@ -27,3 +27,13 @@ def load_plugin_config(): return config['Plugins']['plugins'].split(',') return [] + +def load_config_for_plugin(name): + # Plugins store data using Plugin%PluginName% format. + key = 'Plugin{}'.format(name) + config = load_config() + + if key in config: + return config[key] + + return {} diff --git a/elodie/filesystem.py b/elodie/filesystem.py index f243c206..8a71ab5f 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -532,7 +532,7 @@ def process_file(self, _file, destination, media, **kwargs): # Run `before()` for every loaded plugin and if any of them raise an exception # then we skip importing the file and log a message. - plugins_run_before_status = self.plugins.run_all_before(_file, destination, media) + plugins_run_before_status = self.plugins.run_all_before(_file, destination) if(plugins_run_before_status == False): log.warn('At least one plugin pre-run failed for %s' % _file) return @@ -594,6 +594,14 @@ def process_file(self, _file, destination, media, **kwargs): db.add_hash(checksum, dest_path) db.update_hash_db() + # Run `after()` for every loaded plugin and if any of them raise an exception + # then we skip importing the file and log a message. + plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, metadata) + if(plugins_run_after_status == False): + log.warn('At least one plugin pre-run failed for %s' % _file) + return + + return dest_path def set_utime_from_metadata(self, metadata, file_path): diff --git a/elodie/plugins/dummy/dummy.py b/elodie/plugins/dummy/dummy.py index 6c5eff16..4dd220f9 100644 --- a/elodie/plugins/dummy/dummy.py +++ b/elodie/plugins/dummy/dummy.py @@ -4,7 +4,6 @@ .. moduleauthor:: Jaisen Mathai """ from __future__ import print_function -from builtins import object from elodie.plugins.plugins import PluginBase @@ -16,6 +15,6 @@ class Dummy(PluginBase): def __init__(self): self.before_ran = False - def before(self, file_path, destination_path, media): + def before(self, file_path, destination_folder): self.before_ran = True diff --git a/elodie/plugins/googlephotos/Readme.markdown b/elodie/plugins/googlephotos/Readme.markdown new file mode 100644 index 00000000..7953754d --- /dev/null +++ b/elodie/plugins/googlephotos/Readme.markdown @@ -0,0 +1,63 @@ +# Google Photos Plugin for Elodie + +[![Build Status](https://travis-ci.org/jmathai/elodie.svg?branch=master)](https://travis-ci.org/jmathai/elodie) [![Coverage Status](https://coveralls.io/repos/github/jmathai/elodie/badge.svg?branch=master)](https://coveralls.io/github/jmathai/elodie?branch=master) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/jmathai/elodie/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/jmathai/elodie/?branch=master) + +This plugin uploads all photos imported using Elodie to Google Photos. It was created after [Google Photos and Google Drive synchronization was deprecated](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/). It aims to replicate my [workflow using Google Photos, Google Drive and Elodie](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf). + +I didn't intend on it, but it turned out that with this plugin you can use Google Photos with Google Drive, iCloud Drive, Dropbox or no cloud storage service while still using Google Photos for viewing and experiencing your photo library. + +The hardest part of using this plugin is setting it up. Let's get started. + +# Installation and Setup + +## Google Photos +Let's start by making sure you have a Google Photos account. If you don't, you should start by [creating your Google Photos account](https://photos.google.com/login). + +## Google APIs +Once you've got your Google Photos account created we can enable Google Photos' APIs for your account. + +In order to enable Google APIs you need what's called a project. Don't worry about what it is, just create one so you can enable the Google Photos API for it. +1. Go to [Google's developer console](https://console.developers.google.com). +2. If you have a project already then you can skip this step. + + If you don't already have a project or would like to create one just for this purpose then you should create it now. In the top bar there's a **project selector** which will open a dialog with a button to create a new project. +3. Now you'll need to [enable the Google Photos API for your project](https://console.developers.google.com/apis/library/photoslibrary.googleapis.com). You should be able to follow that link and click the **Enable API** button. Make sure the project from the prior step is selected. +4. Once you've enabled the Google Photos API you will need to [create an OAuth client ID](https://console.developers.google.com/apis/credentials). + 1. Select **other** as the type of client. + 2. Set up a consent screen if needed. Only you'll be seeing this so put whatever you want into the required fields. Most everything can be left blank. + 3. Download the credentials when prompted or click the download icon on the [credentials page](https://console.developers.google.com/apis/credentials). + +## Configure the Google Photos Plugin for Elodie +Now that you're set up with your Google Photos account, have enabled the APIs and configured your OAuth client we're ready to enable this plugin for Elodie. + +1. Move the credentials file you downloaded to a permanent location and update your `config.ini` file. You'll need to add a `[Plugins]` section. + + [Plugins] + plugins=GooglePhotos + + [PluginGooglePhotos] + secrets_file=/full/path/to/saved/secrets_file.json + auth_file=/full/path/to/save/auth_file.json + + I put `secrets_file.json` (the one you downloaded) in my `~/.elodie` directory. `auth_file.json` will be automatically created so make sure the path is writable by the user running `elodie.py`. +2. If you did everything exactly correct you should be able to authenticate Elodie to start uploading to Google Photos. + 1. Start by importing a new photo. + 2. Run `./elodie.py batch` which should open your browser. + 3. Login and tell Google Photos to allow Elodie the requested permissions to your Google Photos account. + 4. At some point you'll likely see a scary warning screen. This is because your OAuth client is not approved but go ahead and click on **Advanced** and **Go to {Your OAuth client name (unsafe)**. + 5. Return to your terminal and close your browser tab if you'd like. + +Assuming you did not see any errors you can go back to your browser and load up Google Photos. If your photos show up in Google Photos then you got everything to work *a lot* easier than I did. + +## Automating It All +I'm not going to go into how you can automate this process but much of it is covered by various blog posts I've done in the past. + +* [Understanding My Need for an Automated Photo Workflow](https://medium.com/vantage/understanding-my-need-for-an-automated-photo-workflow-a2ff95b46f8f#.dmwyjlc57) +* [Introducing Elodie; Your Personal EXIF-based Photo and Video Assistant](https://medium.com/@jmathai/introducing-elodie-your-personal-exif-based-photo-and-video-assistant-d92868f302ec) +* [My Automated Photo Workflow using Google Photos and Elodie](https://medium.com/swlh/my-automated-photo-workflow-using-google-photos-and-elodie-afb753b8c724) +* [One Year of Using an Automated Photo Organization and Archiving Workflow](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf) + +## Credits +Elodie is an open source project with many [contributors](https://github.com/jmathai/elodie/graphs/contributors) and [users](https://github.com/jmathai/elodie/stargazers) who have reported lots of [bugs and feature requests](https://github.com/jmathai/elodie/issues?utf8=%E2%9C%93&q=). + +Google Photos is an amazing product. Kudos to the team for making it so magical. diff --git a/elodie/plugins/googlephotos/__init__.py b/elodie/plugins/googlephotos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/elodie/plugins/googlephotos/googlephotos.py b/elodie/plugins/googlephotos/googlephotos.py new file mode 100644 index 00000000..2437ea8e --- /dev/null +++ b/elodie/plugins/googlephotos/googlephotos.py @@ -0,0 +1,159 @@ +""" +Google Photos plugin object. +This plugin will queue imported photos into the plugin's database file. +Using this plugin should have no impact on performance of importing photos. + +In order to upload the photos to Google Photos you need to run the following command. + +``` + ./elodie.py batch +``` + +That command will execute the batch() method on all plugins, including this one. +This plugin's batch() function reads all files from the database file and attempts to + upload them to Google Photos. +This plugin does not aim to keep Google Photos in sync. +Once a photo is uploaded it's removed from the database and no records are kept thereafter. + +Upload code adapted from https://github.com/eshmu/gphotos-upload + +.. moduleauthor:: Jaisen Mathai +""" +from __future__ import print_function + +import json + +from os.path import basename, isfile + +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import AuthorizedSession +from google.oauth2.credentials import Credentials + +from elodie.media.photo import Photo +from elodie.media.video import Video +from elodie.plugins.plugins import PluginBase + +class GooglePhotos(PluginBase): + """A class to execute plugin actions. + + Requires a config file with the following configurations set. + secrets_file: + The full file path where to find the downloaded secrets. + auth_file: + The full file path where to store authenticated tokens. + + """ + + __name__ = 'GooglePhotos' + + def __init__(self): + super(GooglePhotos, self).__init__() + self.upload_url = 'https://photoslibrary.googleapis.com/v1/uploads' + self.media_create_url = 'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate' + self.scopes = [ + 'https://www.googleapis.com/auth/photoslibrary', + 'https://www.googleapis.com/auth/photoslibrary.appendonly', + 'https://www.googleapis.com/auth/photoslibrary.sharing' + ] + + self.secrets_file = None + if('secrets_file' in self.config_for_plugin): + self.secrets_file = self.config_for_plugin['secrets_file'] + # 'client_id.json' + self.auth_file = None + if('auth_file' in self.config_for_plugin): + self.auth_file = self.config_for_plugin['auth_file'] + self.session = None + + def after(self, file_path, destination_folder, final_file_path, metadata): + extension = metadata['extension'] + if(extension in Photo.extensions or extension in Video.extensions): + self.log(u'Added {} to db.'.format(final_file_path)) + self.db.set(final_file_path, metadata['original_name']) + else: + self.log(u'Skipping {} which is not a supported media type.'.format(final_file_path)) + + def batch(self): + queue = self.db.get_all() + status = True + count = 0 + for key in queue: + this_status = self.upload(key) + if(this_status): + # Remove from queue if successful then increment count + self.db.delete(key) + count = count + 1 + self.display('{} uploaded successfully.'.format(key)) + else: + status = False + self.display('{} failed to upload.'.format(key)) + return (status, count) + + def before(self, file_path, destination_folder): + pass + + def set_session(self): + # Try to load credentials from an auth file. + # If it doesn't exist or is not valid then catch the + # exception and reauthenticate. + try: + creds = Credentials.from_authorized_user_file(self.auth_file, self.scopes) + except: + try: + flow = InstalledAppFlow.from_client_secrets_file(self.secrets_file, self.scopes) + creds = flow.run_local_server() + cred_dict = { + 'token': creds.token, + 'refresh_token': creds.refresh_token, + 'id_token': creds.id_token, + 'scopes': creds.scopes, + 'token_uri': creds.token_uri, + 'client_id': creds.client_id, + 'client_secret': creds.client_secret + } + + # Store the returned authentication tokens to the auth_file. + with open(self.auth_file, 'w') as f: + f.write(json.dumps(cred_dict)) + except: + return + + self.session = AuthorizedSession(creds) + self.session.headers["Content-type"] = "application/octet-stream" + self.session.headers["X-Goog-Upload-Protocol"] = "raw" + + def upload(self, path_to_photo): + self.set_session() + if(self.session is None): + self.log('Could not initialize session') + return None + + self.session.headers["X-Goog-Upload-File-Name"] = basename(path_to_photo) + if(not isfile(path_to_photo)): + self.log('Could not find file: {}'.format(path_to_photo)) + return None + + with open(path_to_photo, 'rb') as f: + photo_bytes = f.read() + + upload_token = self.session.post(self.upload_url, photo_bytes) + if(upload_token.status_code != 200 or not upload_token.content): + self.log('Uploading media failed: ({}) {}'.format(upload_token.status_code, upload_token.content)) + return None + + create_body = json.dumps({'newMediaItems':[{'description':'','simpleMediaItem':{'uploadToken':upload_token.content.decode()}}]}, indent=4) + resp = self.session.post(self.media_create_url, create_body).json() + if( + 'newMediaItemResults' not in resp or + 'status' not in resp['newMediaItemResults'][0] or + 'message' not in resp['newMediaItemResults'][0]['status'] or + ( + resp['newMediaItemResults'][0]['status']['message'] != 'Success' and # photos + resp['newMediaItemResults'][0]['status']['message'] != 'OK' # videos + ) + + ): + self.log('Creating new media item failed: {}'.format(resp['newMediaItemResults'][0]['status'])) + return None + + return resp['newMediaItemResults'][0] diff --git a/requirements-google.txt b/elodie/plugins/googlephotos/requirements.txt similarity index 100% rename from requirements-google.txt rename to elodie/plugins/googlephotos/requirements.txt diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py index 5720355f..bbf16f81 100644 --- a/elodie/plugins/plugins.py +++ b/elodie/plugins/plugins.py @@ -6,28 +6,118 @@ from __future__ import print_function from builtins import object +import io + +from json import dumps, loads from importlib import import_module +from os.path import dirname, dirname, isdir, isfile +from os import mkdir from sys import exc_info from traceback import format_exc -from elodie.config import load_plugin_config +from elodie.compatability import _bytes +from elodie.config import load_config_for_plugin, load_plugin_config +from elodie.constants import application_directory from elodie import log + class ElodiePluginError(Exception): + """Exception which can be thrown by plugins to return failures. + """ pass class PluginBase(object): - + """Base class which all plugins should inherit from. + Defines stubs for all methods and exposes logging and database functionality + """ __name__ = 'PluginBase' - def log(self, msg): - log.info(msg) + def __init__(self): + # Loads the config for the plugin from config.ini + self.config_for_plugin = load_config_for_plugin(self.__name__) + self.db = PluginDb(self.__name__) + + def after(self, file_path, destination_folder, final_file_path, metadata): + pass + def batch(self): + pass + + def before(self, file_path, destination_folder): + pass + + def log(self, msg): + # Writes an info log not shown unless being run in --debug mode. + log.info(dumps( + {self.__name__: msg} + )) + + def display(self, msg): + # Writes a log for all modes and will be displayed. + log.all(dumps( + {self.__name__: msg} + )) + +class PluginDb(object): + """A database module which provides a simple key/value database. + The database is a JSON file located at %application_directory%/plugins/%pluginname.lower()%.json + """ + def __init__(self, plugin_name): + self.db_file = '{}/plugins/{}.json'.format( + application_directory, + plugin_name.lower() + ) + + # If the plugin db directory does not exist, create it + if(not isdir(dirname(self.db_file))): + mkdir(dirname(self.db_file)) + + # If the db file does not exist we initialize it + if(not isfile(self.db_file)): + with io.open(self.db_file, 'wb') as f: + f.write(_bytes(dumps({}))) + + + def get(self, key): + with io.open(self.db_file, 'r') as f: + db = loads(f.read()) + + if(key not in db): + return None + + return db[key] + + def set(self, key, value): + with io.open(self.db_file, 'r') as f: + data = f.read() + db = loads(data) + + db[key] = value + new_content = dumps(db, ensure_ascii=False).encode('utf8') + with io.open(self.db_file, 'wb') as f: + f.write(new_content) + + def get_all(self): + with io.open(self.db_file, 'r') as f: + db = loads(f.read()) + return db + + def delete(self, key): + with io.open(self.db_file, 'r') as f: + db = loads(f.read()) + + # delete key without throwing an exception + db.pop(key, None) + new_content = dumps(db, ensure_ascii=False).encode('utf8') + with io.open(self.db_file, 'wb') as f: + f.write(new_content) class Plugins(object): - """A class to execute plugin actions.""" + """Plugin object which manages all interaction with plugins. + Exposes methods to load plugins and execute their methods. + """ def __init__(self): self.plugins = [] @@ -61,11 +151,55 @@ def load(self): self.loaded = True + def run_all_after(self, file_path, destination_folder, final_file_path, metadata): + """Process `before` methods of each plugin that was loaded. + """ + self.load() + pass_status = True + for cls in self.classes: + this_method = getattr(self.classes[cls], 'after') + # We try to call the plugin's `before()` method. + # If the method explicitly raises an ElodiePluginError we'll fail the import + # by setting pass_status to False. + # If any other error occurs we log the message and proceed as usual. + # By default, plugins don't change behavior. + try: + this_method(file_path, destination_folder, final_file_path, metadata) + log.info('Called after() for {}'.format(cls)) + except ElodiePluginError as err: + log.warn('Plugin {} raised an exception in run_all_before: {}'.format(cls, err)) + log.error(format_exc()) + log.error('false') + pass_status = False + except: + log.error(format_exc()) + return pass_status - def run_all_before(self, file_path, destination_path, media): + def run_batch(self): self.load() + pass_status = True + for cls in self.classes: + this_method = getattr(self.classes[cls], 'batch') + # We try to call the plugin's `before()` method. + # If the method explicitly raises an ElodiePluginError we'll fail the import + # by setting pass_status to False. + # If any other error occurs we log the message and proceed as usual. + # By default, plugins don't change behavior. + try: + this_method() + log.info('Called batch() for {}'.format(cls)) + except ElodiePluginError as err: + log.warn('Plugin {} raised an exception in run_batch: {}'.format(cls, err)) + log.error(format_exc()) + pass_status = False + except: + log.error(format_exc()) + return pass_status + + def run_all_before(self, file_path, destination_folder): """Process `before` methods of each plugin that was loaded. """ + self.load() pass_status = True for cls in self.classes: this_method = getattr(self.classes[cls], 'before') @@ -75,9 +209,10 @@ def run_all_before(self, file_path, destination_path, media): # If any other error occurs we log the message and proceed as usual. # By default, plugins don't change behavior. try: - this_method(file_path, destination_path, media) + this_method(file_path, destination_folder) + log.info('Called before() for {}'.format(cls)) except ElodiePluginError as err: - log.warn('Plugin {} raised an exception: {}'.format(cls, err)) + log.warn('Plugin {} raised an exception in run_all_after: {}'.format(cls, err)) log.error(format_exc()) pass_status = False except: diff --git a/elodie/plugins/runtimeerror/runtimeerror.py b/elodie/plugins/runtimeerror/runtimeerror.py index 8468b1c2..cad00e4f 100644 --- a/elodie/plugins/runtimeerror/runtimeerror.py +++ b/elodie/plugins/runtimeerror/runtimeerror.py @@ -4,7 +4,6 @@ .. moduleauthor:: Jaisen Mathai """ from __future__ import print_function -from builtins import object from elodie.plugins.plugins import PluginBase @@ -16,6 +15,11 @@ class RuntimeError(PluginBase): def __init__(self): pass - def before(self, file_path, destination_path, media): + def after(self, file_path, destination_folder, final_file_path, metadata): print(does_not_exist) + def batch(self): + print(does_not_exist) + + def before(self, file_path, destination_folder): + print(does_not_exist) diff --git a/elodie/plugins/throwerror/throwerror.py b/elodie/plugins/throwerror/throwerror.py index 7b44f1d3..feb7ec64 100644 --- a/elodie/plugins/throwerror/throwerror.py +++ b/elodie/plugins/throwerror/throwerror.py @@ -4,7 +4,6 @@ .. moduleauthor:: Jaisen Mathai """ from __future__ import print_function -from builtins import object from elodie.plugins.plugins import PluginBase, ElodiePluginError @@ -16,5 +15,11 @@ class ThrowError(PluginBase): def __init__(self): pass - def before(self, file_path, destination_path, media): - raise ElodiePluginError('Sample plugin error') + def after(self, file_path, destination_folder, final_file_path, metadata): + raise ElodiePluginError('Sample plugin error for after') + + def batch(self): + raise ElodiePluginError('Sample plugin error for batch') + + def before(self, file_path, destination_folder): + raise ElodiePluginError('Sample plugin error for before') diff --git a/elodie/tests/elodie_test.py b/elodie/tests/elodie_test.py index ddbdf0b9..0ea387b3 100644 --- a/elodie/tests/elodie_test.py +++ b/elodie/tests/elodie_test.py @@ -23,6 +23,8 @@ from elodie.media.photo import Photo from elodie.media.text import Text from elodie.media.video import Video +from elodie.plugins.plugins import Plugins +from elodie.plugins.googlephotos.googlephotos import GooglePhotos os.environ['TZ'] = 'GMT' @@ -605,43 +607,43 @@ def test_verify_error(): assert origin in result.output, result.output assert 'Error 1' in result.output, result.output -# @mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir()) -# def test_cli_batch_plugin_googlephotos(): -# auth_file = helper.get_file('plugins/googlephotos/auth_file.json') -# secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json') -# config_string = """ -# [Plugins] -# plugins=GooglePhotos - -# [PluginGooglePhotos] -# auth_file={} -# secrets_file={} -# """ -# config_string_fmt = config_string.format( -# auth_file, -# secrets_file -# ) -# with open('%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir(), 'w') as f: -# f.write(config_string_fmt) - -# if hasattr(load_config, 'config'): -# del load_config.config - -# final_file_path_1 = helper.get_file('plain.jpg') -# final_file_path_2 = helper.get_file('no-exif.jpg') -# sample_metadata_1 = Photo(final_file_path_1).get_metadata() -# sample_metadata_2 = Photo(final_file_path_2).get_metadata() -# gp = GooglePhotos() -# gp.after('', '', final_file_path_1, sample_metadata_1) -# gp.after('', '', final_file_path_2, sample_metadata_1) - -# if hasattr(load_config, 'config'): -# del load_config.config - -# runner = CliRunner() -# result = runner.invoke(elodie._batch) -# assert "elodie/elodie/tests/files/plain.jpg uploaded successfully.\"}\n" in result.output, result.output -# assert "elodie/elodie/tests/files/no-exif.jpg uploaded successfully.\"}\n" in result.output, result.output +@mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir()) +def test_cli_batch_plugin_googlephotos(): + auth_file = helper.get_file('plugins/googlephotos/auth_file.json') + secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json') + config_string = """ + [Plugins] + plugins=GooglePhotos + + [PluginGooglePhotos] + auth_file={} + secrets_file={} + """ + config_string_fmt = config_string.format( + auth_file, + secrets_file + ) + with open('%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path_1 = helper.get_file('plain.jpg') + final_file_path_2 = helper.get_file('no-exif.jpg') + sample_metadata_1 = Photo(final_file_path_1).get_metadata() + sample_metadata_2 = Photo(final_file_path_2).get_metadata() + gp = GooglePhotos() + gp.after('', '', final_file_path_1, sample_metadata_1) + gp.after('', '', final_file_path_2, sample_metadata_1) + + if hasattr(load_config, 'config'): + del load_config.config + + runner = CliRunner() + result = runner.invoke(elodie._batch) + assert "elodie/elodie/tests/files/plain.jpg uploaded successfully.\"}\n" in result.output, result.output + assert "elodie/elodie/tests/files/no-exif.jpg uploaded successfully.\"}\n" in result.output, result.output def test_cli_debug_import(): runner = CliRunner() diff --git a/elodie/tests/files/plugins/googlephotos/auth_file.json b/elodie/tests/files/plugins/googlephotos/auth_file.json new file mode 100644 index 00000000..d2fd25b1 --- /dev/null +++ b/elodie/tests/files/plugins/googlephotos/auth_file.json @@ -0,0 +1 @@ +{"scopes": ["https://www.googleapis.com/auth/photoslibrary", "https://www.googleapis.com/auth/photoslibrary.appendonly", "https://www.googleapis.com/auth/photoslibrary.sharing"], "id_token": null, "token": "ya29.Gls1B3ymBdURb3tBLAUJgxQtfTzKry0eaUqplbkHxfIYJH9sWvkalLwXprfAW-Ku0Fz8aP3jz7NrncJ2h58idSEgrYXPQ14iVSkwgUGE2gnsxZM6w4TbHz8ny8Yf", "client_id": "1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com", "token_uri": "https://oauth2.googleapis.com/token", "client_secret": "p84GOD_i7_PwDGbAmpnbwKiH", "refresh_token": "1/iHiK9Vbq9i5ebXScyQPDZf9_GJrDOWMvu-2zG9wRsDA"} \ No newline at end of file diff --git a/elodie/tests/files/plugins/googlephotos/secrets_file.json b/elodie/tests/files/plugins/googlephotos/secrets_file.json new file mode 100644 index 00000000..9e5170e0 --- /dev/null +++ b/elodie/tests/files/plugins/googlephotos/secrets_file.json @@ -0,0 +1 @@ +{"installed":{"client_id":"1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com","project_id":"elodietestphotos-1561522235041","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"p84GOD_i7_PwDGbAmpnbwKiH","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file diff --git a/elodie/tests/plugins/__init__.py b/elodie/tests/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/elodie/tests/plugins/googlephotos/__init__.py b/elodie/tests/plugins/googlephotos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/elodie/tests/plugins/googlephotos/googlephotos_test.py b/elodie/tests/plugins/googlephotos/googlephotos_test.py new file mode 100644 index 00000000..16098f13 --- /dev/null +++ b/elodie/tests/plugins/googlephotos/googlephotos_test.py @@ -0,0 +1,185 @@ +from __future__ import absolute_import +# Project imports +import mock +import os +import sys +from tempfile import gettempdir + +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) + +import helper +from elodie.config import load_config +from elodie.plugins.googlephotos.googlephotos import GooglePhotos +from elodie.media.audio import Audio +from elodie.media.photo import Photo + +# Globals to simplify mocking configs +auth_file = helper.get_file('plugins/googlephotos/auth_file.json') +secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json') +config_string = """ +[Plugins] +plugins=GooglePhotos + +[PluginGooglePhotos] +auth_file={} +secrets_file={} + """ +config_string_fmt = config_string.format( + auth_file, + secrets_file +) + +sample_photo = Photo(helper.get_file('plain.jpg')) +sample_metadata = sample_photo.get_metadata() +sample_metadata['original_name'] = 'foobar' + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-set-session' % gettempdir()) +def test_googlephotos_set_session(): + with open('%s/config.ini-googlephotos-set-session' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + assert gp.session is None, gp.session + gp.set_session() + assert gp.session is not None, gp.session + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-after-supported' % gettempdir()) +def test_googlephotos_after_supported(): + with open('%s/config.ini-googlephotos-after-supported' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path = helper.get_file('plain.jpg') + gp = GooglePhotos() + gp.after('', '', final_file_path, sample_metadata) + db_row = gp.db.get(final_file_path) + + if hasattr(load_config, 'config'): + del load_config.config + + assert db_row == 'foobar', db_row + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-after-unsupported' % gettempdir()) +def test_googlephotos_after_unsupported(): + with open('%s/config.ini-googlephotos-after-unsupported' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path = helper.get_file('audio.m4a') + sample_photo = Audio(final_file_path) + sample_metadata = sample_photo.get_metadata() + sample_metadata['original_name'] = 'foobar' + gp = GooglePhotos() + gp.after('', '', final_file_path, sample_metadata) + db_row = gp.db.get(final_file_path) + + if hasattr(load_config, 'config'): + del load_config.config + + assert db_row == None, db_row + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload' % gettempdir()) +def test_googlephotos_upload(): + with open('%s/config.ini-googlephotos-upload' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + gp.set_session() + status = gp.upload(helper.get_file('plain.jpg')) + + assert status is not None, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload-session-fail' % gettempdir()) +def test_googlephotos_upload_session_fail(): + with open('%s/config.ini-googlephotos-upload-session-fail' % gettempdir(), 'w') as f: + f.write(config_string) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + gp.set_session() + status = gp.upload(helper.get_file('plain.jpg')) + + assert status is None, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload-invalid-empty' % gettempdir()) +def test_googlephotos_upload_invalid_empty(): + with open('%s/config.ini-googlephotos-upload-invalid-empty' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + gp.set_session() + status = gp.upload(helper.get_file('invalid.jpg')) + + assert status is None, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload-dne' % gettempdir()) +def test_googlephotos_upload_dne(): + with open('%s/config.ini-googlephotos-upload-dne' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + gp = GooglePhotos() + + if hasattr(load_config, 'config'): + del load_config.config + + gp.set_session() + status = gp.upload('/file/does/not/exist') + + assert status is None, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-batch' % gettempdir()) +def test_googlephotos_batch(): + with open('%s/config.ini-googlephotos-batch' % gettempdir(), 'w') as f: + f.write(config_string_fmt) + if hasattr(load_config, 'config'): + del load_config.config + + final_file_path = helper.get_file('plain.jpg') + gp = GooglePhotos() + gp.after('', '', final_file_path, sample_metadata) + db_row = gp.db.get(final_file_path) + assert db_row == 'foobar', db_row + + status, count = gp.batch() + db_row_after = gp.db.get(final_file_path) + assert status == True, status + assert count == 1, count + assert db_row_after is None, db_row_after + + + if hasattr(load_config, 'config'): + del load_config.config + + + gp.set_session() + status = gp.upload(helper.get_file('invalid.jpg')) + + assert status is None, status diff --git a/elodie/tests/plugins_test.py b/elodie/tests/plugins_test.py index 15cfd708..9f36010e 100644 --- a/elodie/tests/plugins_test.py +++ b/elodie/tests/plugins_test.py @@ -9,7 +9,7 @@ from . import helper from elodie.config import load_config -from elodie.plugins.plugins import Plugins +from elodie.plugins.plugins import Plugins, PluginBase, PluginDb @mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-unset-backwards-compat' % gettempdir()) def test_load_plugins_unset_backwards_compat(): @@ -134,7 +134,7 @@ def test_run_before(): plugins = Plugins() plugins.load() before_ran_1 = plugins.classes['Dummy'].before_ran - plugins.run_all_before('', '', '') + plugins.run_all_before('', '') before_ran_2 = plugins.classes['Dummy'].before_ran if hasattr(load_config, 'config'): @@ -155,12 +155,16 @@ def test_throw_error(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '', '') + status_after = plugins.run_all_after('', '', '', '') + status_batch = plugins.run_batch() + status_before = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status == False, status + assert status_after == False, status_after + assert status_batch == False, status_batch + assert status_before == False, status_before @mock.patch('elodie.config.config_file', '%s/config.ini-throw-error-one-of-many' % gettempdir()) def test_throw_error_one_of_many(): @@ -174,12 +178,16 @@ def test_throw_error_one_of_many(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '', '') + status_after = plugins.run_all_after('', '', '', '') + status_batch = plugins.run_batch() + status_before = plugins.run_all_before('', '') if hasattr(load_config, 'config'): del load_config.config - assert status == False, status + assert status_after == False, status_after + assert status_batch == False, status_batch + assert status_before == False, status_before @mock.patch('elodie.config.config_file', '%s/config.ini-throw-runtime-error' % gettempdir()) def test_throw_error_runtime_error(): @@ -193,9 +201,55 @@ def test_throw_error_runtime_error(): plugins = Plugins() plugins.load() - status = plugins.run_all_before('', '', '') - - if hasattr(load_config, 'config'): - del load_config.config - - assert status == True, status + status_after = plugins.run_all_after('', '', '', '') + status_batch = plugins.run_batch() + status_before = plugins.run_all_before('', '') + + if hasattr(load_config, 'config'): + del load_config.config + + assert status_after == True, status_after + assert status_batch == True, status_batch + assert status_before == True, status_before + +def test_plugin_base_inherits_db(): + plugin_base = PluginBase() + assert hasattr(plugin_base.db, 'get') + assert hasattr(plugin_base.db, 'set') + assert hasattr(plugin_base.db, 'get_all') + assert hasattr(plugin_base.db, 'delete') + +def test_db_initialize_file(): + db = PluginDb('foobar') + try: + os.remove(db.db_file) + except OSError: + pass + db = PluginDb('foobar') + +def test_db_get_then_set_then_get_then_delete(): + db = PluginDb('foobar') + foo = db.get('foo') + assert foo is None, foo + db.set('foo', 'bar') + foo = db.get('foo') + assert foo == 'bar', foo + db.delete('foo') + foo = db.get('foo') + assert foo is None, foo + +def test_db_get_all(): + # we initialize the db to get the file path to delete then reinitialize + db = PluginDb('foobar') + try: + os.remove(db.db_file) + except OSError: + pass + db = PluginDb('foobar') + db.set('a', '1') + db.set('b', '2') + db.set('c', '3') + db.set('d', '4') + all_rows = db.get_all() + + assert all_rows == {'a': '1', 'b': '2', 'c': '3', 'd': '4'}, all_rows