DB Connections, Thread based processing, Other updates..

config.py

@@ -0,0 +1,17 @@
+import os
+# Credentials you get from registering a new application
+client_id = os.environ.get("CLIENT_ID")
+client_secret = os.environ.get("CLIENT_SECRET")
+#redirect_uri = "https://nimpf.akimbo.com"
+# OAuth endpoints given in the Google API documentation
+authorization_base_url = "https://accounts.google.com/o/oauth2/v2/auth"
+token_url = "https://www.googleapis.com/oauth2/v4/token"
+refresh_url = token_url
+scope = [
+    "https://www.googleapis.com/auth/userinfo.email",
+    "https://www.googleapis.com/auth/userinfo.profile",
+    "openid",
+    "https://www.googleapis.com/auth/gmail.settings.basic",
+    "https://www.googleapis.com/auth/gmail.modify",
+]
+whitelist = ['sethgodin.com', 'adobe.com', 'alxpck.com', 'altmba.com']

db.py

@@ -0,0 +1,39 @@
+import psycopg2, os
+import psycopg2.extras
+
+class Db():
+    def __init__(self):
+        self._conn_str = os.environ.get('DATABASE_URL')
+
+    def query(self, sql, data=None, verbose=False):
+        conn = psycopg2.connect(self._conn_str)
+        cur = conn.cursor(cursor_factory = psycopg2.extras.DictCursor)
+        if verbose == True:
+            print(cur.mogrify(sql, data))
+        cur.execute(sql, data)
+        conn.commit()
+
+        if cur.description is not None:
+            results = cur.fetchall()
+
+        cur.close()
+        conn.close()
+
+        if cur.description is not None:
+            if len(results) == 1:
+                return results[0]
+            elif len(results) == 0:
+                return None
+            else:
+                return results
+
+
+
+
+# UNIT TESTS
+if __name__=='__main__':
+    db = Db()
+    users = db.query('select * from participant;')
+    print(users)
+    #users = db.query('insert into participant (email) values (%s) returning email;', ['shandgp@clarkson.edu'])
+    #print(users)

main.py

@@ -2,79 +2,27 @@
 import json, csv, os, requests, re
 from datetime import datetime, timedelta
 from time import time
-from flask import Flask, request, url_for, redirect, render_template, session, flash, get_flashed_messages
+from flask import Flask, request, url_for, redirect, render_template, session, flash, get_flashed_messages, abort
 from flask_sslify import SSLify
+from config import *
+from user import User
+from worker import conn
+
 
 app = Flask(__name__)
 app.secret_key = os.environ.get("SECRET_KEY")
 app.permanent_session_lifetime = timedelta(days=365)
 
-# Credentials you get from registering a new application
-client_id = os.environ.get("CLIENT_ID")
-client_secret = os.environ.get("CLIENT_SECRET")
-#redirect_uri = "https://nimpf.akimbo.com"
-# OAuth endpoints given in the Google API documentation
-authorization_base_url = "https://accounts.google.com/o/oauth2/v2/auth"
-token_url = "https://www.googleapis.com/oauth2/v4/token"
-refresh_url = token_url
-scope = [
-    "https://www.googleapis.com/auth/userinfo.email",
-    "https://www.googleapis.com/auth/userinfo.profile",
-    "openid",
-    "https://www.googleapis.com/auth/gmail.settings.basic",
-    "https://www.googleapis.com/auth/gmail.modify",
-]
-
-def refresh_token():
-    google = OAuth2Session(client_id, token=session['token'])
-    extra = {
-        'client_id': client_id,
-        'client_secret': client_secret,
-    }
-    session['token'] = google.refresh_token(refresh_url, **extra)
-    session.modified = True
-    print('token updated!')
-    return google
-
-def remove_label(message_id):
-    google = OAuth2Session(client_id, token=session['token'])
-    if session['token']['expires_at'] < time()+10:
-        google = refresh_token()
-    params = {"removeLabelIds": ['CATEGORY_PROMOTIONS'], "addLabelIds": ['CATEGORY_PERSONAL']}
-    headers = {"Content-Type": "application/json"}
-    r = google.post("https://www.googleapis.com/gmail/v1/users/me/messages/{}/modify".format(message_id), data=json.dumps(params), headers=headers)
-    return r.text
-
-def create_filter(whitelist):
-    google = OAuth2Session(client_id, token=session['token'])
-    if session['token']['expires_at'] < time()+10:
-        google = refresh_token()
-    headers = {"Content-Type": "application/json"}
-    params = {
-        "criteria": {
-            "from": " OR ".join(whitelist)
-        },
-        "action": {
-            "removeLabelIds": ["SPAM"],
-            "addLabelIds": ["CATEGORY_PERSONAL"]
-        }
-    }
-    r = google.post("https://www.googleapis.com/gmail/v1/users/me/settings/filters", data=json.dumps(params), headers=headers)
-    return r.text
-
-whitelist = ['sethgodin.com', 'adobe.com']
 
 @app.route("/")
 def index():
     print("on / {}".format(session))
     if 'logged_in' in session.keys() and session['logged_in'] == True:
-        #session['token']['expires_at'] = time() - 10
-        google = OAuth2Session(client_id, token=session['token'])
-        if session['token']['expires_at'] < time()+10:
-            google = refresh_token()
-        r = google.get('https://www.googleapis.com/oauth2/v1/userinfo')
-        #print(r.json())
-        data = r.json()
+        u = User(session['user'])
+        #u.get_by_email(u.email()) # refresh data
+        data = u.user_info()
+        session['user'] = u.json() # save any updates to session cookie
+        session.modified = True
         email, img, name = data['email'], data['picture'], data['name']
         return render_template('index.html', email=email, img=img, name=name)
     else:
@@ -84,26 +32,31 @@ def index():
             # Fetch the access token
             token = google.fetch_token(token_url, client_secret=client_secret,
                     authorization_response=redirect_response)
-            session['token'] = token
-            session.modified = True
-
             # Fetch a protected resource, i.e. user profile
             r = google.get('https://www.googleapis.com/oauth2/v1/userinfo')
+            data = r.json()
+            email, name = data['email'], data['name']
+            u = User()
+            if u.get_by_email(email) is None:
+                print('creating new user')
+                if u.create(email, name) == False:
+                    print('failed to create new user')
+                    abort(500)
+            u.set_token(token)
+
+            session['user'] = u.json()
             session['logged_in'] = True
             session.modified = True
             return redirect('/')
         else:
             return redirect('/login')
 
-
-
 @app.route("/login")
 def login():
     print("on /login {}".format(session))
     session['redirect_uri'] = request.url_root.rstrip('/login')
     session.modified = True
     google = OAuth2Session(client_id, scope=scope, redirect_uri=session['redirect_uri'])
-
     # Redirect user to Google for authorization
     authorization_url, state = google.authorization_url(authorization_base_url,
         # offline for refresh token
@@ -117,34 +70,28 @@ def login():
 @app.route('/process')
 def process():
     print("on /process {}".format(session))
-    google = OAuth2Session(client_id, token=session['token'])
-    if session['token']['expires_at'] < time()+10:
-        google = refresh_token()
-    r = google.get("https://www.googleapis.com/gmail/v1/users/me/messages?labelIds=CATEGORY_PROMOTIONS")
-    promo_messages = r.json()['messages']
-    for i, row in enumerate(promo_messages):
-        r = google.get("https://www.googleapis.com/gmail/v1/users/me/messages/{}".format(row['id']))
-        message = r.json()
-        for val in message['payload']['headers']:
-            if val['name'] == 'From':
-                sender = val['value']
-                string = re.compile("@(.+)>")
-                match = re.search(string, sender)
-                domain = match.group(1)
-                for whitelisted_domain in whitelist:
-                    if whitelisted_domain in domain: # gracefully handle subdomains
-                        print(domain, 'removed')
-                        remove_label(row['id'])
-
-    flash('Promo folder cleaned')
+    u = User(session['user'])
+    if u.filter_made() == True:
+        flash('Your inbox filter has already been created', 'info')
+    else:
+        """
+        result1 = u.clear_promo_folder()
+        if result1 == True:
+            result2 = u.make_filter()
+            if result2 == True:
+                flash('Success', 'success')
+            else:
+                flash('Error making filter', 'danger')
+        else:
+            flash('Error cleaning promo folder', 'danger')
+        """
+        #job = q.enqueue(process_user, u.json())
+        job_id = u.go()
+        flash('Success', 'success')
+    session['user'] = u.json() 
+    session.modified = True
     return redirect(url_for('index'))
 
-@app.route('/process2')
-def process2():
-    print("on /process2 {}".format(session))
-    print(create_filter(whitelist))
-    flash('Filter made')
-    return redirect(url_for('index'))
 
 @app.route('/clear')
 def clear():
@@ -153,5 +100,14 @@ def clear():
     session.modified = True
     return redirect('/')
 
+@app.route('/test')
+def test():
+    print("on /test {}".format(session))
+    print(query())
+    job = q.enqueue(query)
+    return redirect('/')
+
+
 if __name__ == '__main__':
-    app.run(ssl_context='adhoc')
+    app.run(ssl_context='adhoc')
+

requirements.txt

@@ -1,46 +1,22 @@
-anytree==2.7.1
-APScheduler==3.5.0
-asn1crypto==0.24.0
-Authlib==0.10
-certifi==2017.7.27.1
-cffi==1.12.2
+certifi==2019.9.11
+cffi==1.13.0
 chardet==3.0.4
-click==6.7
-cryptography==2.6.1
-fastcache==1.1.0
-Flask==0.12.2
+Click==7.0
+cryptography==2.8
+Flask==1.1.1
 Flask-SSLify==0.1.5
-googlemaps==3.1.3
-gspread==3.1.0
-gunicorn==19.7.1
-httplib2==0.14.0
-idna==2.6
-itsdangerous==0.24
-Jinja2==2.9.6
-MarkupSafe==1.0
-newrelic==5.0.2.126
-numpy==1.16.4
-oauth2client==4.1.3
+idna==2.8
+itsdangerous==1.1.0
+Jinja2==2.10.3
+MarkupSafe==1.1.1
 oauthlib==3.1.0
-pandas==0.25.1
-phonenumbers==8.9.12
-psycopg2==2.7.3.2
-pyasn1==0.4.7
-pyasn1-modules==0.2.6
+psycopg2==2.8.3
 pycparser==2.19
-PyMySQL==0.9.3
 pyOpenSSL==19.0.0
-python-dateutil==2.8.0
-pytz==2017.3
-redis==2.10.6
+redis==3.3.11
 requests==2.22.0
 requests-oauthlib==1.2.0
-rq==0.12.0
-rsa==4.0
-six==1.11.0
-stripe==2.29.3
-toml==0.9.6
-tzlocal==1.5.1
-untangle==1.1.1
-urllib3==1.22
-Werkzeug==0.12.2
+rq==1.1.0
+six==1.12.0
+urllib3==1.25.6
+Werkzeug==0.16.0

templates/index.html

@@ -4,29 +4,46 @@
 </head>
 <body style='height: 100%'>
     <div class='container'>
-        {% for message in get_flashed_messages() %}
-        <div class='alert alert-success alert-dismissible fade show'>
-            {{message}}
-            <button type="button" class="close" data-dismiss="alert" aria-label="Close">
-                <span aria-hidden="true">&times;</span>
-            </button>
-        </div>
-        {% endfor %}
     </div>
     <div class='container' style='display: flex; justify-content: center; align-items: center; height: 100vh'>
-        <div class='card col-sm-6' style='vertical-align: middle; height: 300px; border-radius: 2em;'>  
+        <div class='card col-sm-6' style='vertical-align: middle; height: auto;'>  
             <div class='card-body'>
+                {% for category, message in get_flashed_messages(with_categories=true) %}
+                <div class='alert alert-{{category}} alert-dismissible fade show'>
+                    {{message}}
+                    <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+                        <span aria-hidden="true">&times;</span>
+                    </button>
+                </div>
+                {% endfor %}
                 <div class='text-center'>
                     <img src="{{img}}" style='display: inline; width:100; margin-bottom: 2em; border-radius: 50%;'/>
                     <h4>{{name | upper}}</h4>
                     <p class='lead'>{{email | lower}}</p>
-                    <a href='/process' class='card-link'>Pull Existing</a>
-                    <a href='/process2' class='card-link'>Build Filter</a>
+                    <a href='/process' class='card-link' id='link'>Go</a>
                 </div>
             </div>
         </div>
     </div>
 
+    <script type="text/javascript">
+        let link = document.querySelector('#link')
+        var spinner = function() {
+            var card = link.parentElement.parentElement;
+            const div = document.createElement('div');
+            div.setAttribute('class', 'd-flex justify-content-center')
+            const spinner = document.createElement('div');
+            spinner.setAttribute('class', 'spinner-border text-center');
+            spinner.setAttribute('role', 'status');
+            spinner.innerHTML = "<span class='sr-only'>Loading...</span>"
+            div.appendChild(spinner)
+            card.appendChild(div)
+        }
+
+        link.addEventListener('click', spinner);
+    </script>
+
+
     <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
     <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>