Merge branch 'feat/evernote-integration' into develop

README.md

@@ -95,7 +95,20 @@ Visit `https://[project-id].appspot.com` to see the app live.
 * Flash card widget for spreadsheet access (e.g. random quotes, excerpts)
 * Export all data to CSV
 
-## Google Home Integration
+## Integrations
+
+All integrations work out of the box on flowdash.co, but if you're spinning up your own instance, you'll need to set up each integration you need.  See below for specific instructions.
+
+### Pocket
+
+Create an app at https://getpocket.com/developer/ and update secrets.POCKET_CONSUMER_KEY
+
+### Evernote
+
+1. Request an API Key at https://dev.evernote.com
+2. Request a webhook at https://dev.evernote.com/support/ pointing to [Your Domain]/api/integrations/evernote/webhook
+
+### Google Home Integration
 
 We've used API.AI to create an agent that integrates with Google Actions / Assistant / Home. To connect Assistant with a new instance of Flow:
 
@@ -106,10 +119,11 @@ We've used API.AI to create an agent that integrates with Google Actions / Assis
 5. Go to integrations and add and authorize 'Actions on Google'
 6. Preview the integration using the web preview
 
-## Facebook Messenger Integration
+### Facebook Messenger Integration
 
 The messenger bot lives at https://www.facebook.com/FlowDashboard/
 
+To create a new messenger bot for your own instance of Flow, see the Facebook quickstart: https://developers.facebook.com/docs/messenger-platform/guides/quick-start
 
 ## Planned Features
 

api.py

@@ -1,7 +1,8 @@
 
 from datetime import datetime, timedelta, time
 from models import Project, Habit, HabitDay, Goal, MiniJournal, User, Task, \
-    Readable, TrackingDay, Event, JournalTag, Report
+    Readable, TrackingDay, Event, JournalTag, Report, Quote
+from constants import READABLE
 from google.appengine.ext import ndb
 import authorized
 import handlers
@@ -371,7 +372,16 @@ class ReadableAPI(handlers.JsonRequestHandler):
 
     @authorized.role('user')
     def list(self, d):
-        readables = Readable.Unread(self.user)
+        page, max, offset = tools.paging_params(self.request)
+        favorites = self.request.get_range('favorites') == 1
+        with_notes = self.request.get_range('with_notes') == 1
+        unread = self.request.get_range('unread') == 1
+        read = self.request.get_range('read') == 1
+        since = self.request.get('since')  # ISO
+        readables = Readable.Fetch(self.user, favorites=favorites,
+                                   unread=unread, read=read,
+                                   with_notes=with_notes, since=since,
+                                   limit=max, offset=offset)
         self.set_response({
             'readables': [r.json() for r in readables]
         }, success=True)
@@ -380,8 +390,15 @@ def list(self, d):
     def update(self, d):
         id = self.request.get('id')
         params = tools.gets(self,
+            integers=['type'],
+            strings=['notes', 'title', 'url', 'author', 'source'],
             booleans=['read', 'favorite'])
-        r = Readable.get_by_id(id, parent=self.user.key)
+        logging.debug(params)
+        if id:
+            r = Readable.get_by_id(id, parent=self.user.key)
+        else:
+            # New
+            r = Readable.CreateOrUpdate(self.user, None, **params)
         if r:
             r.Update(**params)
             if r.source == 'pocket':
@@ -398,6 +415,37 @@ def update(self, d):
             'readable': r.json() if r else None
         })
 
+    @authorized.role('user')
+    def batch_create(self, d):
+        readings = json.loads(self.request.get('readings'))
+        source = self.request.get('source', default_value='form')
+        dbp = []
+        for r in readings:
+            type_string = r.get('type')
+            if type_string:
+                r['type'] = READABLE.LOOKUP.get(type_string.lower())
+            r = Readable.CreateOrUpdate(self.user, None, source=source, read=True, **r)
+            dbp.append(r)
+        if dbp:
+            ndb.put_multi(dbp)
+            self.success = True
+            self.message = "Putting %d" % len(dbp)
+        self.set_response()
+
+    @authorized.role('user')
+    def random_batch(self, d):
+        '''
+        Return a random batch, optionally filtered
+        '''
+        BATCH_SIZE = 50
+        sample_keys = Readable.Fetch(self.user, with_notes=True, limit=500, keys_only=True)
+        if len(sample_keys) > BATCH_SIZE:
+            sample_keys = random.sample(sample_keys, BATCH_SIZE)
+        readables = ndb.get_multi(sample_keys)
+        self.set_response({
+            'readables': [r.json() for r in readables]
+            }, success=True)
+
     @authorized.role('user')
     def delete(self, d):
         id = self.request.get('id')
@@ -416,6 +464,67 @@ def delete(self, d):
         self.set_response()
 
 
+class QuoteAPI(handlers.JsonRequestHandler):
+
+    @authorized.role('user')
+    def list(self, d):
+        quotes = Quote.Fetch(self.user)
+        self.set_response({
+            'quotes': [q.json() for q in quotes]
+        }, success=True)
+
+    @authorized.role('user')
+    def update(self, d):
+        id = self.request.get('id')
+        params = tools.gets(self,
+            strings=['source', 'content', 'link', 'location', 'date'],
+            lists=['tags']
+        )
+        logging.debug(params)
+        quote = None
+        if id:
+            quote = Quote.get_by_id(id, parent=self.user.key)
+        else:
+            if 'date' in params:
+                params['dt_added'] = tools.fromISODate(params.get('date'))
+            quote = Quote.Create(self.user, **params)
+            self.message = "Quote saved!" if quote else "Couldn't create quote"
+            self.success = quote is not None
+        quote.Update(tags=params.get('tags'))
+        quote.put()
+        self.set_response({
+            'quote': quote.json() if quote else None
+        })
+
+    @authorized.role('user')
+    def batch_create(self, d):
+        quotes = json.loads(self.request.get('quotes'))
+        dbp = []
+        for q in quotes:
+            if 'dt_added' in q and isinstance(q['dt_added'], basestring):
+                q['dt_added'] = tools.fromISODate(q['dt_added'])
+            q = Quote.Create(self.user, **q)
+            dbp.append(q)
+        if dbp:
+            ndb.put_multi(dbp)
+            self.success = True
+            self.message = "Putting %d" % len(dbp)
+        self.set_response()
+
+    @authorized.role('user')
+    def random_batch(self, d):
+        '''
+        Return a random batch, optionally filtered
+        '''
+        BATCH_SIZE = 50
+        sample_keys = Quote.Fetch(self.user, limit=500, keys_only=True)
+        if len(sample_keys) > BATCH_SIZE:
+            sample_keys = random.sample(sample_keys, BATCH_SIZE)
+        quotes = ndb.get_multi(sample_keys)
+        self.set_response({
+            'quotes': [q.json() for q in quotes]
+            }, success=True)
+
 class JournalTagAPI(handlers.JsonRequestHandler):
 
     @authorized.role('user')
@@ -743,6 +852,82 @@ def pocket_disconnect(self, d):
             'user': self.user.json() if self.user else None
         }, success=True)
 
+    @authorized.role('user')
+    def evernote_authenticate(self, d):
+        '''
+        Step 1
+        '''
+        from services import flow_evernote
+        authorize_url = flow_evernote.get_request_token(self.user, self.request.host_url + "/app/integrations/evernote_connect")
+        self.success = bool(authorize_url)
+        self.set_response(data={
+            'redirect': authorize_url
+            })
+
+    @authorized.role('user')
+    def evernote_authorize(self, d):
+        '''
+        Step 2
+        '''
+        from services import flow_evernote
+        ot = self.request.get('oauth_token')
+        ot_secret = self.request.get('oauth_token_secret')
+        verifier = self.request.get('oauth_verifier')
+        access_token, en_user = flow_evernote.get_access_token(self.user, ot, ot_secret, verifier)
+        logging.debug([access_token, en_user])
+        if access_token:
+            self.user.set_integration_prop('evernote_access_token', access_token)
+            if en_user:
+                self.user.evernote_id = str(en_user.id)
+            self.user.put()
+            self.update_session_user(self.user)
+            # self.session['pocket_code'] = code
+            self.success = True
+        self.set_response(data={
+            'user': self.user.json()
+            })
+
+    @authorized.role('user')
+    def evernote_disconnect(self, d):
+        '''
+        '''
+        self.user.set_integration_prop('evernote_access_token', None)
+        self.user.evernote_id = None
+        self.user.put()
+        self.update_session_user(self.user)
+        self.set_response({
+            'user': self.user.json() if self.user else None
+        }, success=True)
+
+    @authorized.role()
+    def evernote_webhook(self, d):
+        '''
+        Evernote notifies us of a change
+
+        Webhook request for note creation of the form:
+        [base URL]/?userId=[user ID]&guid=[note GUID]&notebookGuid=[notebook GUID]&reason=create
+        '''
+        from services import flow_evernote
+        from models import Quote
+        config_notebook_ids = self.user.get_integration_prop('evernote_notebook_ids').split(',') # Comma sep
+        note_guid = self.request.get('guid')
+        evernote_id = self.request.get('userId')
+        notebook_guid = self.request.get('notebookGuid')
+        data = {}
+        user = User.query().filter(User.evernote_id == evernote_id)
+        if user and (not config_notebook_ids or notebook_guid in config_notebook_ids):
+            title, content = flow_evernote.get_note(note_guid)
+            if title and content:
+                # TODO: Link and Tags
+                q = Quote.Create(user, source=title, content=content)
+                q.put()
+                self.success = True
+            else:
+                self.message = "Failed ot parse note"
+        else:
+            logging.warning("Note from ignored notebook or user not found")
+        self.set_response(data=data)
+
 
 class AgentAPI(handlers.JsonRequestHandler):
 

constants.py

@@ -118,10 +118,18 @@ class READABLE():
     # Type
     ARTICLE = 1
     BOOK = 2
+    PAPER = 3
 
     LABELS = {
         ARTICLE: "Article",
-        BOOK: "Book"
+        BOOK: "Book",
+        PAPER: "Paper"
+    }
+
+    LOOKUP = {
+        "article": 1,
+        "book": 2,
+        "paper": 3
     }
 
 

flow.py

@@ -78,6 +78,12 @@
         webapp2.Route('/api/readable', handler=api.ReadableAPI, handler_method="list", methods=["GET"]),
         webapp2.Route('/api/readable', handler=api.ReadableAPI, handler_method="update", methods=["POST"]),
         webapp2.Route('/api/readable/delete', handler=api.ReadableAPI, handler_method="delete", methods=["POST"]),
+        webapp2.Route('/api/readable/batch', handler=api.ReadableAPI, handler_method="batch_create", methods=["POST"]),
+        webapp2.Route('/api/readable/random', handler=api.ReadableAPI, handler_method="random_batch", methods=["GET"]),
+        webapp2.Route('/api/quote', handler=api.QuoteAPI, handler_method="list", methods=["GET"]),
+        webapp2.Route('/api/quote', handler=api.QuoteAPI, handler_method="update", methods=["POST"]),
+        webapp2.Route('/api/quote/batch', handler=api.QuoteAPI, handler_method="batch_create", methods=["POST"]),
+        webapp2.Route('/api/quote/random', handler=api.QuoteAPI, handler_method="random_batch", methods=["GET"]),
         webapp2.Route('/api/analysis', handler=api.AnalysisAPI, handler_method="get", methods=["GET"]),
         webapp2.Route('/api/journaltag', handler=api.JournalTagAPI, handler_method="list", methods=["GET"]),
         webapp2.Route('/api/report', handler=api.ReportAPI, handler_method="list", methods=["GET"]),
@@ -97,6 +103,10 @@
         webapp2.Route('/api/integrations/pocket/authenticate', handler=api.IntegrationsAPI, handler_method="pocket_authenticate", methods=["POST"]),
         webapp2.Route('/api/integrations/pocket/authorize', handler=api.IntegrationsAPI, handler_method="pocket_authorize", methods=["POST"]),
         webapp2.Route('/api/integrations/pocket/disconnect', handler=api.IntegrationsAPI, handler_method="pocket_disconnect", methods=["POST"]),
+        webapp2.Route('/api/integrations/evernote/authenticate', handler=api.IntegrationsAPI, handler_method="evernote_authenticate", methods=["POST"]),
+        webapp2.Route('/api/integrations/evernote/authorize', handler=api.IntegrationsAPI, handler_method="evernote_authorize", methods=["POST"]),
+        webapp2.Route('/api/integrations/evernote/disconnect', handler=api.IntegrationsAPI, handler_method="evernote_disconnect", methods=["POST"]),
+        webapp2.Route('/api/integrations/evernote/webhook', handler=api.IntegrationsAPI, handler_method="evernote_webhook", methods=["POST"]),
 
         # Agent
         webapp2.Route('/api/agent/apiai/request', handler=api.AgentAPI, handler_method="apiai_request", methods=["POST"]),

index.yaml

@@ -97,3 +97,36 @@ indexes:
 # manually, move them above the marker line.  The index.yaml file is
 # automatically uploaded to the admin console when you next deploy
 # your application using appcfg.py.
+
+- kind: Quote
+  ancestor: yes
+  properties:
+  - name: dt_added
+    direction: desc
+
+- kind: Readable
+  ancestor: yes
+  properties:
+  - name: dt_added
+    direction: desc
+
+- kind: Readable
+  ancestor: yes
+  properties:
+  - name: favorite
+  - name: dt_added
+    direction: desc
+
+- kind: Readable
+  ancestor: yes
+  properties:
+  - name: has_notes
+  - name: dt_added
+    direction: desc
+
+- kind: Readable
+  ancestor: yes
+  properties:
+  - name: read
+  - name: dt_read
+    direction: desc