Add a plugin to upload photos to Google Photos (#319)

Fixes #315.

This PR aims to address the [recent changes](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/) in Google Photos + Google Drive where syncing between the two is no longer supported.

It works by uploading photos as part of the import process to add a copy of every photo in your library to Google Photos. Google Drive is not required for this plugin to work.

This plugin lets you have all your photos in Google Photos without relying on Google Drive. You can use another cloud storage service like iCloud or Dropbox or no cloud storage at all.

- [x] Add tests for `after()` plugin methods.
- [x] Add support for storage/async support.
- [x] Include plugins into code coverage.
- [x] Sweep code and clean up and add comments.

.travis.yml

@@ -10,6 +10,7 @@ before_install:
   - "sudo apt-get install python-dev python-pip -y"
 install:
   - "pip install -r elodie/tests/requirements.txt"
+  - "pip install -r elodie/plugins/googlephotos/requirements.txt"
   - "pip install coveralls"
 before_script:
   - "mkdir ~/.elodie"

Readme.md

@@ -260,7 +260,7 @@ You can configure how Elodie names your files using placeholders. This works sim
 
 If you'd like to specify your own naming convention it's recommended you include something that's mostly unique like the time including seconds. You'll need to include a `[File]` section in your `config.ini` file with a name attribute. If a placeholder doesn't have a value then it plus any preceding characters which are not alphabetic are removed.
 
-By default the resulting filename is all lowercased. To change this behavior to upppercasing add capitalization=upper.
+By default the resulting filename is all lowercased. To change this behavior to uppercasing add capitalization=upper.
 
 ```
 [File]
@@ -270,7 +270,7 @@ name=%date-%original_name-%title.jpg
 
 date=%Y-%m-%b-%H-%M-%S
 name=%date-%original_name-%album.jpg
-capitalization=uppper
+capitalization=upper
 # -> 2012-05-MAR-12-59-30-DSC_1234-MY-ALBUM.JPG
 ```
 

elodie.py

@@ -27,6 +27,7 @@
 from elodie.media.audio import Audio
 from elodie.media.photo import Photo
 from elodie.media.video import Video
+from elodie.plugins.plugins import Plugins
 from elodie.result import Result
 
 
@@ -70,6 +71,16 @@ def import_file(_file, destination, album_from_folder, trash, allow_duplicates):
 
     return dest_path or None
 
+@click.command('batch')
+@click.option('--debug', default=False, is_flag=True,
+              help='Override the value in constants.py with True.')
+def _batch(debug):
+    """Run batch() for all plugins.
+    """
+    constants.debug = debug
+    plugins = Plugins()
+    plugins.run_batch()
+
 
 @click.command('import')
 @click.option('--destination', type=click.Path(file_okay=False),
@@ -340,6 +351,7 @@ def main():
 main.add_command(_update)
 main.add_command(_generate_db)
 main.add_command(_verify)
+main.add_command(_batch)
 
 
 if __name__ == '__main__':

elodie/compatability.py

@@ -24,6 +24,11 @@ def _decode(string, encoding=sys.getfilesystemencoding()):
 
     return string
 
+def _bytes(string):
+    if constants.python_version == 3:
+        return bytes(string, 'utf8')
+    else:
+        return bytes(string)
 
 def _copyfile(src, dst):
     # shutil.copy seems slow, changing to streaming according to

elodie/config.py

@@ -27,3 +27,13 @@ def load_plugin_config():
         return config['Plugins']['plugins'].split(',')
 
     return []
+
+def load_config_for_plugin(name):
+    # Plugins store data using Plugin%PluginName% format.
+    key = 'Plugin{}'.format(name)
+    config = load_config()
+
+    if key in config:
+        return config[key]
+
+    return {}

elodie/filesystem.py

@@ -532,7 +532,7 @@ def process_file(self, _file, destination, media, **kwargs):
 
         # Run `before()` for every loaded plugin and if any of them raise an exception
         #  then we skip importing the file and log a message.
-        plugins_run_before_status = self.plugins.run_all_before(_file, destination, media)
+        plugins_run_before_status = self.plugins.run_all_before(_file, destination)
         if(plugins_run_before_status == False):
             log.warn('At least one plugin pre-run failed for %s' % _file)
             return
@@ -594,6 +594,14 @@ def process_file(self, _file, destination, media, **kwargs):
         db.add_hash(checksum, dest_path)
         db.update_hash_db()
 
+        # Run `after()` for every loaded plugin and if any of them raise an exception
+        #  then we skip importing the file and log a message.
+        plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, metadata)
+        if(plugins_run_after_status == False):
+            log.warn('At least one plugin pre-run failed for %s' % _file)
+            return
+
+
         return dest_path
 
     def set_utime_from_metadata(self, metadata, file_path):

elodie/plugins/dummy/dummy.py

@@ -4,7 +4,6 @@
 .. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
 """
 from __future__ import print_function
-from builtins import object
 
 from elodie.plugins.plugins import PluginBase
 
@@ -16,6 +15,6 @@ class Dummy(PluginBase):
     def __init__(self):
         self.before_ran = False
 
-    def before(self, file_path, destination_path, media):
+    def before(self, file_path, destination_folder):
         self.before_ran = True
 

elodie/plugins/googlephotos/Readme.markdown

@@ -0,0 +1,63 @@
+# Google Photos Plugin for Elodie
+
+[![Build Status](https://travis-ci.org/jmathai/elodie.svg?branch=master)](https://travis-ci.org/jmathai/elodie) [![Coverage Status](https://coveralls.io/repos/github/jmathai/elodie/badge.svg?branch=master)](https://coveralls.io/github/jmathai/elodie?branch=master) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/jmathai/elodie/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/jmathai/elodie/?branch=master)
+
+This plugin uploads all photos imported using Elodie to Google Photos. It was created after [Google Photos and Google Drive synchronization was deprecated](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/). It aims to replicate my [workflow using Google Photos, Google Drive and Elodie](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf).
+
+I didn't intend on it, but it turned out that with this plugin you can use Google Photos with Google Drive, iCloud Drive, Dropbox or no cloud storage service while still using Google Photos for viewing and experiencing your photo library.
+
+The hardest part of using this plugin is setting it up. Let's get started.
+
+# Installation and Setup
+
+## Google Photos
+Let's start by making sure you have a Google Photos account. If you don't, you should start by [creating your Google Photos account](https://photos.google.com/login).
+
+## Google APIs
+Once you've got your Google Photos account created we can enable Google Photos' APIs for your account.
+
+In order to enable Google APIs you need what's called a project. Don't worry about what it is, just create one so you can enable the Google Photos API for it.
+1. Go to [Google's developer console](https://console.developers.google.com).
+2. If you have a project already then you can skip this step.
+
+    If you don't already have a project or would like to create one just for this purpose then you should create it now. In the top bar there's a **project selector** which will open a dialog with a button to create a new project.
+3. Now you'll need to [enable the Google Photos API for your project](https://console.developers.google.com/apis/library/photoslibrary.googleapis.com). You should be able to follow that link and click the **Enable API** button. Make sure the project from the prior step is selected.
+4. Once you've enabled the Google Photos API you will need to [create an OAuth client ID](https://console.developers.google.com/apis/credentials).
+    1. Select **other** as the type of client.
+    2. Set up a consent screen if needed. Only you'll be seeing this so put whatever you want into the required fields. Most everything can be left blank.
+    3. Download the credentials when prompted or click the download icon on the [credentials page](https://console.developers.google.com/apis/credentials).
+
+## Configure the Google Photos Plugin for Elodie
+Now that you're set up with your Google Photos account, have enabled the APIs and configured your OAuth client we're ready to enable this plugin for Elodie.
+
+1. Move the credentials file you downloaded to a permanent location and update your `config.ini` file. You'll need to add a `[Plugins]` section.
+
+          [Plugins]
+          plugins=GooglePhotos
+        
+          [PluginGooglePhotos]
+          secrets_file=/full/path/to/saved/secrets_file.json
+          auth_file=/full/path/to/save/auth_file.json
+
+    I put `secrets_file.json` (the one you downloaded) in my `~/.elodie` directory. `auth_file.json` will be automatically created so make sure the path is writable by the user running `elodie.py`.
+2. If you did everything exactly correct you should be able to authenticate Elodie to start uploading to Google Photos.
+    1. Start by importing a new photo.
+    2. Run `./elodie.py batch` which should open your browser.
+    3. Login and tell Google Photos to allow Elodie the requested permissions to your Google Photos account.
+    4. At some point you'll likely see a scary warning screen. This is because your OAuth client is not approved but go ahead and click on **Advanced** and **Go to {Your OAuth client name (unsafe)**.
+    5. Return to your terminal and close your browser tab if you'd like.
+
+Assuming you did not see any errors you can go back to your browser and load up Google Photos. If your photos show up in Google Photos then you got everything to work *a lot* easier than I did.
+
+## Automating It All
+I'm not going to go into how you can automate this process but much of it is covered by various blog posts I've done in the past.
+
+* [Understanding My Need for an Automated Photo Workflow](https://medium.com/vantage/understanding-my-need-for-an-automated-photo-workflow-a2ff95b46f8f#.dmwyjlc57)
+* [Introducing Elodie; Your Personal EXIF-based Photo and Video Assistant](https://medium.com/@jmathai/introducing-elodie-your-personal-exif-based-photo-and-video-assistant-d92868f302ec)
+* [My Automated Photo Workflow using Google Photos and Elodie](https://medium.com/swlh/my-automated-photo-workflow-using-google-photos-and-elodie-afb753b8c724)
+* [One Year of Using an Automated Photo Organization and Archiving Workflow](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf)
+
+## Credits
+Elodie is an open source project with many [contributors](https://github.com/jmathai/elodie/graphs/contributors) and [users](https://github.com/jmathai/elodie/stargazers) who have reported lots of [bugs and feature requests](https://github.com/jmathai/elodie/issues?utf8=%E2%9C%93&q=).
+
+Google Photos is an amazing product. Kudos to the team for making it so magical.

elodie/plugins/googlephotos/googlephotos.py

@@ -0,0 +1,159 @@
+"""
+Google Photos plugin object.
+This plugin will queue imported photos into the plugin's database file.
+Using this plugin should have no impact on performance of importing photos.
+
+In order to upload the photos to Google Photos you need to run the following command.
+
+```
+    ./elodie.py batch
+```
+
+That command will execute the batch() method on all plugins, including this one.
+This plugin's batch() function reads all files from the database file and attempts to 
+    upload them to Google Photos.
+This plugin does not aim to keep Google Photos in sync.
+Once a photo is uploaded it's removed from the database and no records are kept thereafter.
+
+Upload code adapted from https://github.com/eshmu/gphotos-upload
+
+.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
+"""
+from __future__ import print_function
+
+import json
+
+from os.path import basename, isfile
+
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import AuthorizedSession
+from google.oauth2.credentials import Credentials
+
+from elodie.media.photo import Photo
+from elodie.media.video import Video
+from elodie.plugins.plugins import PluginBase
+
+class GooglePhotos(PluginBase):
+    """A class to execute plugin actions.
+       
+       Requires a config file with the following configurations set.
+       secrets_file:
+            The full file path where to find the downloaded secrets.
+       auth_file:
+            The full file path where to store authenticated tokens.
+    
+    """
+
+    __name__ = 'GooglePhotos'
+
+    def __init__(self):
+        super(GooglePhotos, self).__init__()
+        self.upload_url = 'https://photoslibrary.googleapis.com/v1/uploads'
+        self.media_create_url = 'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate'
+        self.scopes = [
+            'https://www.googleapis.com/auth/photoslibrary',
+            'https://www.googleapis.com/auth/photoslibrary.appendonly',
+            'https://www.googleapis.com/auth/photoslibrary.sharing'
+        ]
+
+        self.secrets_file = None
+        if('secrets_file' in self.config_for_plugin):
+            self.secrets_file = self.config_for_plugin['secrets_file']
+        # 'client_id.json'
+        self.auth_file = None
+        if('auth_file' in self.config_for_plugin):
+            self.auth_file = self.config_for_plugin['auth_file']
+        self.session = None
+
+    def after(self, file_path, destination_folder, final_file_path, metadata):
+        extension = metadata['extension']
+        if(extension in Photo.extensions or extension in Video.extensions):
+            self.log(u'Added {} to db.'.format(final_file_path))
+            self.db.set(final_file_path, metadata['original_name'])
+        else:
+            self.log(u'Skipping {} which is not a supported media type.'.format(final_file_path))
+
+    def batch(self):
+        queue = self.db.get_all()
+        status = True
+        count = 0
+        for key in queue:
+            this_status = self.upload(key)
+            if(this_status):
+                # Remove from queue if successful then increment count
+                self.db.delete(key)
+                count = count + 1
+                self.display('{} uploaded successfully.'.format(key))
+            else:
+                status = False
+                self.display('{} failed to upload.'.format(key))
+        return (status, count)
+
+    def before(self, file_path, destination_folder):
+        pass
+
+    def set_session(self):
+        # Try to load credentials from an auth file.
+        # If it doesn't exist or is not valid then catch the 
+        #  exception and reauthenticate.
+        try:
+            creds = Credentials.from_authorized_user_file(self.auth_file, self.scopes)
+        except:
+            try:
+                flow = InstalledAppFlow.from_client_secrets_file(self.secrets_file, self.scopes)
+                creds = flow.run_local_server()
+                cred_dict = {
+                    'token': creds.token,
+                    'refresh_token': creds.refresh_token,
+                    'id_token': creds.id_token,
+                    'scopes': creds.scopes,
+                    'token_uri': creds.token_uri,
+                    'client_id': creds.client_id,
+                    'client_secret': creds.client_secret
+                }
+
+                # Store the returned authentication tokens to the auth_file.
+                with open(self.auth_file, 'w') as f:
+                    f.write(json.dumps(cred_dict))
+            except:
+                return
+
+        self.session = AuthorizedSession(creds)
+        self.session.headers["Content-type"] = "application/octet-stream"
+        self.session.headers["X-Goog-Upload-Protocol"] = "raw"
+
+    def upload(self, path_to_photo):
+        self.set_session()
+        if(self.session is None):
+            self.log('Could not initialize session')
+            return None
+
+        self.session.headers["X-Goog-Upload-File-Name"] = basename(path_to_photo)
+        if(not isfile(path_to_photo)):
+            self.log('Could not find file: {}'.format(path_to_photo))
+            return None
+
+        with open(path_to_photo, 'rb') as f:
+            photo_bytes = f.read()
+
+        upload_token = self.session.post(self.upload_url, photo_bytes)
+        if(upload_token.status_code != 200 or not upload_token.content):
+            self.log('Uploading media failed: ({}) {}'.format(upload_token.status_code, upload_token.content))
+            return None
+
+        create_body = json.dumps({'newMediaItems':[{'description':'','simpleMediaItem':{'uploadToken':upload_token.content.decode()}}]}, indent=4)
+        resp = self.session.post(self.media_create_url, create_body).json()
+        if(
+            'newMediaItemResults' not in resp or
+            'status' not in resp['newMediaItemResults'][0] or
+            'message' not in resp['newMediaItemResults'][0]['status'] or
+            (
+                resp['newMediaItemResults'][0]['status']['message'] != 'Success' and # photos
+                resp['newMediaItemResults'][0]['status']['message'] != 'OK' # videos
+            )
+
+        ):
+            self.log('Creating new media item failed: {}'.format(resp['newMediaItemResults'][0]['status']))
+            return None
+
+        return resp['newMediaItemResults'][0]