Merge a3dd355 into 9e42edf

.travis.yml

@@ -10,6 +10,7 @@ before_install:
   - "sudo apt-get install python-dev python-pip -y"
 install:
   - "pip install -r elodie/tests/requirements.txt"
+  - "pip install -r elodie/plugins/googlephotos/requirements.txt"
   - "pip install coveralls"
 before_script:
   - "mkdir ~/.elodie"

elodie.py

@@ -27,6 +27,7 @@
 from elodie.media.audio import Audio
 from elodie.media.photo import Photo
 from elodie.media.video import Video
+from elodie.plugins.plugins import Plugins
 from elodie.result import Result
 
 
@@ -70,6 +71,14 @@ def import_file(_file, destination, album_from_folder, trash, allow_duplicates):
 
     return dest_path or None
 
+@click.command('batch')
+@click.option('--debug', default=False, is_flag=True,
+              help='Override the value in constants.py with True.')
+def _batch(debug):
+    constants.debug = debug
+    plugins = Plugins()
+    plugins.run_batch()
+
 
 @click.command('import')
 @click.option('--destination', type=click.Path(file_okay=False),
@@ -340,6 +349,7 @@ def main():
 main.add_command(_update)
 main.add_command(_generate_db)
 main.add_command(_verify)
+main.add_command(_batch)
 
 
 if __name__ == '__main__':

elodie/compatability.py

@@ -24,6 +24,11 @@ def _decode(string, encoding=sys.getfilesystemencoding()):
 
     return string
 
+def _bytes(string):
+    if constants.python_version == 3:
+        return bytes(string, 'utf8')
+    else:
+        return bytes(string)
 
 def _copyfile(src, dst):
     # shutil.copy seems slow, changing to streaming according to

elodie/config.py

@@ -27,3 +27,13 @@ def load_plugin_config():
         return config['Plugins']['plugins'].split(',')
 
     return []
+
+def load_config_for_plugin(name):
+    # Plugins store data using Plugin%PluginName% format.
+    key = 'Plugin{}'.format(name)
+    config = load_config()
+
+    if key in config:
+        return config[key]
+
+    return {}

elodie/filesystem.py

@@ -532,7 +532,7 @@ def process_file(self, _file, destination, media, **kwargs):
 
         # Run `before()` for every loaded plugin and if any of them raise an exception
         #  then we skip importing the file and log a message.
-        plugins_run_before_status = self.plugins.run_all_before(_file, destination, media)
+        plugins_run_before_status = self.plugins.run_all_before(_file, destination)
         if(plugins_run_before_status == False):
             log.warn('At least one plugin pre-run failed for %s' % _file)
             return
@@ -594,6 +594,14 @@ def process_file(self, _file, destination, media, **kwargs):
         db.add_hash(checksum, dest_path)
         db.update_hash_db()
 
+        # Run `after()` for every loaded plugin and if any of them raise an exception
+        #  then we skip importing the file and log a message.
+        plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, metadata)
+        if(plugins_run_after_status == False):
+            log.warn('At least one plugin pre-run failed for %s' % _file)
+            return
+
+
         return dest_path
 
     def set_utime_from_metadata(self, metadata, file_path):

elodie/media/media.py

@@ -128,6 +128,7 @@ def get_exiftool_attributes(self):
 
         with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et:
             metadata = et.get_metadata(source)
+            print(metadata)
             if not metadata:
                 return False
 

elodie/plugins/dummy/dummy.py

@@ -4,7 +4,6 @@
 .. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
 """
 from __future__ import print_function
-from builtins import object
 
 from elodie.plugins.plugins import PluginBase
 
@@ -16,6 +15,6 @@ class Dummy(PluginBase):
     def __init__(self):
         self.before_ran = False
 
-    def before(self, file_path, destination_path, media):
+    def before(self, file_path, destination_folder):
         self.before_ran = True
 

elodie/plugins/googlephotos/googlephotos.py

@@ -0,0 +1,141 @@
+"""
+Google Photos plugin object.
+Upload code adapted from https://github.com/eshmu/gphotos-upload
+
+.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
+"""
+from __future__ import print_function
+
+import json
+
+from os.path import basename, isfile
+
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import AuthorizedSession
+from google.oauth2.credentials import Credentials
+
+from elodie.media.photo import Photo
+from elodie.media.video import Video
+from elodie.plugins.plugins import PluginBase
+
+class GooglePhotos(PluginBase):
+    """A class to execute plugin actions.
+       
+       Requires a config file with the following configurations set.
+       secrets_file:
+            The full file path where to find the downloaded secrets.
+       auth_file:
+            The full file path where to store authenticated tokens.
+    
+    """
+
+    __name__ = 'GooglePhotos'
+
+    def __init__(self):
+        super(GooglePhotos, self).__init__()
+        self.upload_url = 'https://photoslibrary.googleapis.com/v1/uploads'
+        self.media_create_url = 'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate'
+        self.scopes = [
+            'https://www.googleapis.com/auth/photoslibrary',
+            'https://www.googleapis.com/auth/photoslibrary.appendonly',
+            'https://www.googleapis.com/auth/photoslibrary.sharing'
+        ]
+
+        self.secrets_file = None
+        if('secrets_file' in self.config_for_plugin):
+            self.secrets_file = self.config_for_plugin['secrets_file']
+        # 'client_id.json'
+        self.auth_file = None
+        if('auth_file' in self.config_for_plugin):
+            self.auth_file = self.config_for_plugin['auth_file']
+        self.session = None
+
+    def after(self, file_path, destination_folder, final_file_path, metadata):
+        extension = metadata['extension']
+        if(extension in Photo.extensions or extension in Video.extensions):
+            self.log(u'Added {} to db.'.format(final_file_path))
+            self.db.set(final_file_path, metadata['original_name'])
+        else:
+            self.log(u'Skipping {} which is not a supported media type.'.format(final_file_path))
+
+    def batch(self):
+        queue = self.db.get_all()
+        status = True
+        count = 0
+        for key in queue:
+            this_status = self.upload(key)
+            if(this_status):
+                # Remove from queue if successful then increment count
+                self.db.delete(key)
+                count = count + 1
+                self.display('{} uploaded successfully.'.format(key))
+            else:
+                status = False
+                self.display('{} failed to upload.'.format(key))
+        return (status, count)
+
+    def before(self, file_path, destination_folder):
+        pass
+
+    def set_session(self):
+        # Try to load credentials from an auth file.
+        # If it doesn't exist or is not valid then catch the 
+        #  exception and reauthenticate.
+        try:
+            creds = Credentials.from_authorized_user_file(self.auth_file, self.scopes)
+        except:
+            flow = InstalledAppFlow.from_client_secrets_file(self.secrets_file, self.scopes)
+            creds = flow.run_local_server()
+            cred_dict = {
+                'token': creds.token,
+                'refresh_token': creds.refresh_token,
+                'id_token': creds.id_token,
+                'scopes': creds.scopes,
+                'token_uri': creds.token_uri,
+                'client_id': creds.client_id,
+                'client_secret': creds.client_secret
+            }
+
+            # Store the returned authentication tokens to the auth_file.
+            with open(self.auth_file, 'w') as f:
+                f.write(json.dumps(cred_dict))
+
+        self.session = AuthorizedSession(creds)
+        self.session.headers["Content-type"] = "application/octet-stream"
+        self.session.headers["X-Goog-Upload-Protocol"] = "raw"
+
+    def upload(self, path_to_photo):
+        self.set_session()
+        if(self.session is None):
+            self.log('Could not initialize session')
+            return None
+
+        self.session.headers["X-Goog-Upload-File-Name"] = basename(path_to_photo)
+        if(not isfile(path_to_photo)):
+            self.log('Could not find file: {}'.format(path_to_photo))
+            return None
+
+        with open(path_to_photo, 'rb') as f:
+            photo_bytes = f.read()
+
+        upload_token = self.session.post(self.upload_url, photo_bytes)
+        if(upload_token.status_code != 200 or not upload_token.content):
+            self.log('Uploading media failed: ({}) {}'.format(upload_token.status_code, upload_token.content))
+            return None
+
+        create_body = json.dumps({'newMediaItems':[{'description':'','simpleMediaItem':{'uploadToken':upload_token.content.decode()}}]}, indent=4)
+        resp = self.session.post(self.media_create_url, create_body).json()
+        if(
+            'newMediaItemResults' not in resp or
+            'status' not in resp['newMediaItemResults'][0] or
+            'message' not in resp['newMediaItemResults'][0]['status'] or
+            (
+                resp['newMediaItemResults'][0]['status']['message'] != 'Success' and # photos
+                resp['newMediaItemResults'][0]['status']['message'] != 'OK' # videos
+            )
+
+        ):
+            self.log('Creating new media item failed: {}'.format(resp['newMediaItemResults'][0]['status']))
+            return None
+
+        return resp['newMediaItemResults'][0]