From f155080c142eaf37d40b173fc5c3ffc89006e56e Mon Sep 17 00:00:00 2001 From: Grayden Shand Date: Fri, 18 Oct 2019 15:44:51 -0400 Subject: [PATCH] DB Connections, Thread based processing, Other updates.. --- __init__.py | 0 config.py | 17 +++ db.py | 39 ++++++ main.py | 146 +++++++------------- requirements.txt | 54 +++----- templates/index.html | 39 ++++-- user.py | 311 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 461 insertions(+), 145 deletions(-) create mode 100644 __init__.py create mode 100644 config.py create mode 100644 db.py create mode 100644 user.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config.py b/config.py new file mode 100644 index 0000000..9d75db3 --- /dev/null +++ b/config.py @@ -0,0 +1,17 @@ +import os +# Credentials you get from registering a new application +client_id = os.environ.get("CLIENT_ID") +client_secret = os.environ.get("CLIENT_SECRET") +#redirect_uri = "https://nimpf.akimbo.com" +# OAuth endpoints given in the Google API documentation +authorization_base_url = "https://accounts.google.com/o/oauth2/v2/auth" +token_url = "https://www.googleapis.com/oauth2/v4/token" +refresh_url = token_url +scope = [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/userinfo.profile", + "openid", + "https://www.googleapis.com/auth/gmail.settings.basic", + "https://www.googleapis.com/auth/gmail.modify", +] +whitelist = ['sethgodin.com', 'adobe.com', 'alxpck.com', 'altmba.com'] diff --git a/db.py b/db.py new file mode 100644 index 0000000..516d7e6 --- /dev/null +++ b/db.py @@ -0,0 +1,39 @@ +import psycopg2, os +import psycopg2.extras + +class Db(): + def __init__(self): + self._conn_str = os.environ.get('DATABASE_URL') + + def query(self, sql, data=None, verbose=False): + conn = psycopg2.connect(self._conn_str) + cur = conn.cursor(cursor_factory = psycopg2.extras.DictCursor) + if verbose == True: + print(cur.mogrify(sql, data)) + cur.execute(sql, data) + conn.commit() + + if cur.description is not None: + results = cur.fetchall() + + cur.close() + conn.close() + + if cur.description is not None: + if len(results) == 1: + return results[0] + elif len(results) == 0: + return None + else: + return results + + + + +# UNIT TESTS +if __name__=='__main__': + db = Db() + users = db.query('select * from participant;') + print(users) + #users = db.query('insert into participant (email) values (%s) returning email;', ['shandgp@clarkson.edu']) + #print(users) diff --git a/main.py b/main.py index 76fed0c..b84be48 100644 --- a/main.py +++ b/main.py @@ -2,79 +2,27 @@ import json, csv, os, requests, re from datetime import datetime, timedelta from time import time -from flask import Flask, request, url_for, redirect, render_template, session, flash, get_flashed_messages +from flask import Flask, request, url_for, redirect, render_template, session, flash, get_flashed_messages, abort from flask_sslify import SSLify +from config import * +from user import User +from worker import conn + app = Flask(__name__) app.secret_key = os.environ.get("SECRET_KEY") app.permanent_session_lifetime = timedelta(days=365) -# Credentials you get from registering a new application -client_id = os.environ.get("CLIENT_ID") -client_secret = os.environ.get("CLIENT_SECRET") -#redirect_uri = "https://nimpf.akimbo.com" -# OAuth endpoints given in the Google API documentation -authorization_base_url = "https://accounts.google.com/o/oauth2/v2/auth" -token_url = "https://www.googleapis.com/oauth2/v4/token" -refresh_url = token_url -scope = [ - "https://www.googleapis.com/auth/userinfo.email", - "https://www.googleapis.com/auth/userinfo.profile", - "openid", - "https://www.googleapis.com/auth/gmail.settings.basic", - "https://www.googleapis.com/auth/gmail.modify", -] - -def refresh_token(): - google = OAuth2Session(client_id, token=session['token']) - extra = { - 'client_id': client_id, - 'client_secret': client_secret, - } - session['token'] = google.refresh_token(refresh_url, **extra) - session.modified = True - print('token updated!') - return google - -def remove_label(message_id): - google = OAuth2Session(client_id, token=session['token']) - if session['token']['expires_at'] < time()+10: - google = refresh_token() - params = {"removeLabelIds": ['CATEGORY_PROMOTIONS'], "addLabelIds": ['CATEGORY_PERSONAL']} - headers = {"Content-Type": "application/json"} - r = google.post("https://www.googleapis.com/gmail/v1/users/me/messages/{}/modify".format(message_id), data=json.dumps(params), headers=headers) - return r.text - -def create_filter(whitelist): - google = OAuth2Session(client_id, token=session['token']) - if session['token']['expires_at'] < time()+10: - google = refresh_token() - headers = {"Content-Type": "application/json"} - params = { - "criteria": { - "from": " OR ".join(whitelist) - }, - "action": { - "removeLabelIds": ["SPAM"], - "addLabelIds": ["CATEGORY_PERSONAL"] - } - } - r = google.post("https://www.googleapis.com/gmail/v1/users/me/settings/filters", data=json.dumps(params), headers=headers) - return r.text - -whitelist = ['sethgodin.com', 'adobe.com'] @app.route("/") def index(): print("on / {}".format(session)) if 'logged_in' in session.keys() and session['logged_in'] == True: - #session['token']['expires_at'] = time() - 10 - google = OAuth2Session(client_id, token=session['token']) - if session['token']['expires_at'] < time()+10: - google = refresh_token() - r = google.get('https://www.googleapis.com/oauth2/v1/userinfo') - #print(r.json()) - data = r.json() + u = User(session['user']) + #u.get_by_email(u.email()) # refresh data + data = u.user_info() + session['user'] = u.json() # save any updates to session cookie + session.modified = True email, img, name = data['email'], data['picture'], data['name'] return render_template('index.html', email=email, img=img, name=name) else: @@ -84,26 +32,31 @@ def index(): # Fetch the access token token = google.fetch_token(token_url, client_secret=client_secret, authorization_response=redirect_response) - session['token'] = token - session.modified = True - # Fetch a protected resource, i.e. user profile r = google.get('https://www.googleapis.com/oauth2/v1/userinfo') + data = r.json() + email, name = data['email'], data['name'] + u = User() + if u.get_by_email(email) is None: + print('creating new user') + if u.create(email, name) == False: + print('failed to create new user') + abort(500) + u.set_token(token) + + session['user'] = u.json() session['logged_in'] = True session.modified = True return redirect('/') else: return redirect('/login') - - @app.route("/login") def login(): print("on /login {}".format(session)) session['redirect_uri'] = request.url_root.rstrip('/login') session.modified = True google = OAuth2Session(client_id, scope=scope, redirect_uri=session['redirect_uri']) - # Redirect user to Google for authorization authorization_url, state = google.authorization_url(authorization_base_url, # offline for refresh token @@ -117,34 +70,28 @@ def login(): @app.route('/process') def process(): print("on /process {}".format(session)) - google = OAuth2Session(client_id, token=session['token']) - if session['token']['expires_at'] < time()+10: - google = refresh_token() - r = google.get("https://www.googleapis.com/gmail/v1/users/me/messages?labelIds=CATEGORY_PROMOTIONS") - promo_messages = r.json()['messages'] - for i, row in enumerate(promo_messages): - r = google.get("https://www.googleapis.com/gmail/v1/users/me/messages/{}".format(row['id'])) - message = r.json() - for val in message['payload']['headers']: - if val['name'] == 'From': - sender = val['value'] - string = re.compile("@(.+)>") - match = re.search(string, sender) - domain = match.group(1) - for whitelisted_domain in whitelist: - if whitelisted_domain in domain: # gracefully handle subdomains - print(domain, 'removed') - remove_label(row['id']) - - flash('Promo folder cleaned') + u = User(session['user']) + if u.filter_made() == True: + flash('Your inbox filter has already been created', 'info') + else: + """ + result1 = u.clear_promo_folder() + if result1 == True: + result2 = u.make_filter() + if result2 == True: + flash('Success', 'success') + else: + flash('Error making filter', 'danger') + else: + flash('Error cleaning promo folder', 'danger') + """ + #job = q.enqueue(process_user, u.json()) + job_id = u.go() + flash('Success', 'success') + session['user'] = u.json() + session.modified = True return redirect(url_for('index')) -@app.route('/process2') -def process2(): - print("on /process2 {}".format(session)) - print(create_filter(whitelist)) - flash('Filter made') - return redirect(url_for('index')) @app.route('/clear') def clear(): @@ -153,5 +100,14 @@ def clear(): session.modified = True return redirect('/') +@app.route('/test') +def test(): + print("on /test {}".format(session)) + print(query()) + job = q.enqueue(query) + return redirect('/') + + if __name__ == '__main__': - app.run(ssl_context='adhoc') \ No newline at end of file + app.run(ssl_context='adhoc') + diff --git a/requirements.txt b/requirements.txt index 74d3c44..a926449 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,46 +1,22 @@ -anytree==2.7.1 -APScheduler==3.5.0 -asn1crypto==0.24.0 -Authlib==0.10 -certifi==2017.7.27.1 -cffi==1.12.2 +certifi==2019.9.11 +cffi==1.13.0 chardet==3.0.4 -click==6.7 -cryptography==2.6.1 -fastcache==1.1.0 -Flask==0.12.2 +Click==7.0 +cryptography==2.8 +Flask==1.1.1 Flask-SSLify==0.1.5 -googlemaps==3.1.3 -gspread==3.1.0 -gunicorn==19.7.1 -httplib2==0.14.0 -idna==2.6 -itsdangerous==0.24 -Jinja2==2.9.6 -MarkupSafe==1.0 -newrelic==5.0.2.126 -numpy==1.16.4 -oauth2client==4.1.3 +idna==2.8 +itsdangerous==1.1.0 +Jinja2==2.10.3 +MarkupSafe==1.1.1 oauthlib==3.1.0 -pandas==0.25.1 -phonenumbers==8.9.12 -psycopg2==2.7.3.2 -pyasn1==0.4.7 -pyasn1-modules==0.2.6 +psycopg2==2.8.3 pycparser==2.19 -PyMySQL==0.9.3 pyOpenSSL==19.0.0 -python-dateutil==2.8.0 -pytz==2017.3 -redis==2.10.6 +redis==3.3.11 requests==2.22.0 requests-oauthlib==1.2.0 -rq==0.12.0 -rsa==4.0 -six==1.11.0 -stripe==2.29.3 -toml==0.9.6 -tzlocal==1.5.1 -untangle==1.1.1 -urllib3==1.22 -Werkzeug==0.12.2 +rq==1.1.0 +six==1.12.0 +urllib3==1.25.6 +Werkzeug==0.16.0 diff --git a/templates/index.html b/templates/index.html index e140b9b..9e88359 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,29 +4,46 @@
- {% for message in get_flashed_messages() %} -
- {{message}} - -
- {% endfor %}
-
+
+ {% for category, message in get_flashed_messages(with_categories=true) %} +
+ {{message}} + +
+ {% endfor %}

{{name | upper}}

{{email | lower}}

- Pull Existing - Build Filter + Go
+ + + diff --git a/user.py b/user.py new file mode 100644 index 0000000..da2d66e --- /dev/null +++ b/user.py @@ -0,0 +1,311 @@ +from db import Db +from datetime import datetime +from time import time, sleep +from config import * +import json, re +from requests_oauthlib import OAuth2Session +from threading import Thread + +class User(): + """ + Data type for users/users of this system + + Public methods: + * create + * get_by_email + * email + * token + * name + * created_at + * filter_made + * make_filter + """ + + def __init__(self, data=None): + if data == None: + self._email = None + self._token = None + self._filter_made = None + self._name = None + self._created_at = None + self._filter_id = None + else: + data = json.loads(data) + self._email = data['email'] + self._token = data['token'] + self._filter_made = data['filter_made'] + self._name = data['name'] + self._created_at = datetime.fromtimestamp(data['created_at']) + if self.token()['expires_at'] < time() + 10: + self.refresh_token() + self._filter_id = data['filter_id'] + + def __repr__(self): + return f"{self.email()} - {self.name()}" + + def create(self, email, name=None, token=None): + created_at = datetime.now() + sql = "INSERT INTO participant (email, name, token, created_at) VALUES (%s, %s, %s, %s);" + data = [email, name, token, created_at] + db = Db() + try: + db.query(sql, data) + self._email = email + self._name = name + self._created_at = created_at + self._token = token + return True + except Exception as e: + print(e) + return False + + + def email(self): + return self._email + + def token(self): + return self._token + + def name(self): + return self._name + + def created_at(self): + return self._created_at + + def filter_made(self): + return self._filter_made + + def filter_id(self): + return self._filter_id + + def get_by_email(self, email): + db = Db() + sql = "SELECT * FROM participant WHERE email = %s;" + data = [email] + participant = db.query(sql, data) + if participant is None: + return None + self._email = participant['email'] + if participant['token'] is not None: + self._token = json.loads(participant['token']) + else: + self._token = None + self._filter_made = participant['filter_made'] + self._name = participant['name'] + self._created_at = participant['created_at'] + self._filter_id = participant['filter_id'] + return self + + def json(self): + _dict = {'email': self.email(), 'name': self.name(), 'token': self.token(), 'filter_made': self.filter_made(), 'created_at': self.created_at().timestamp(), "filter_id": self.filter_id()} + return json.dumps(_dict) + + def make_filter(self): + if self._email is None: + raise Exception('No user specified: use .get_by_email() or .create() first') + if self._token is None: + raise Exception("User's Oauth2 token is None") + + google = OAuth2Session(client_id, token=self.token()) + if self.token()['expires_at'] < time()+10: + google = self.refresh_token() + headers = {"Content-Type": "application/json"} + params = { + "criteria": { + "from": " OR ".join(whitelist) + }, + "action": { + "removeLabelIds": ["SPAM"], + "addLabelIds": ["CATEGORY_PERSONAL"] + } + } + r = google.post("https://www.googleapis.com/gmail/v1/users/me/settings/filters", data=json.dumps(params), headers=headers) + + if r.status_code == 200: + filter_id = r.json()['id'] + db = Db() + sql = "UPDATE participant SET filter_made=%s, filter_id=%s WHERE email = %s;" + data = [True, filter_id, self.email()] + db.query(sql, data, verbose=True) + self._filter_made = True + self._filter_id = filter_id + return True + else: + # TODO -- error handling + print(r.text) + return False + + def refresh_token(self): + google = OAuth2Session(client_id, token=self.token()) + extra = { + 'client_id': client_id, + 'client_secret': client_secret, + } + self.set_token(google.refresh_token(refresh_url, **extra)) + print('token updated!') + return google + + def user_info(self): + google = OAuth2Session(client_id, token=self.token()) + if self.token()['expires_at'] < time() + 10: + google = self.refresh_token() + r = google.get('https://www.googleapis.com/oauth2/v1/userinfo') + data = r.json() + if self._name != data['name']: + db = Db() + sql = 'UPDATE participant SET name = %s WHERE email = %s;' + params = [data['name'], self._email] + db.query(sql, params) + self._name = data['name'] + if self._email != data['email']: + db = Db() + sql = 'UPDATE participant SET name = %s WHERE email = %s;' + params = [data['email'], self._email] + db.query(sql, params) + self._email = data['email'] + return r.json() + + def set_token(self, token): + if self._email is None: + raise Exception('Anonymous user, set email first') + db = Db() + sql = 'UPDATE participant set token = %s where email = %s;' + data = [json.dumps(token), self.email()] + db.query(sql, data) + self._token = token + return self + + def get_messages(self): + if self.token() is None: + raise Exception("User's Oauth2 token is None") + google = OAuth2Session(client_id, token=self.token()) + if self.token()['expires_at'] < time()+10: + google = self.refresh_token() + r = google.get("https://www.googleapis.com/gmail/v1/users/me/messages?labelIds=CATEGORY_PROMOTIONS") + return r.json()['messages'] + + def get_message(self, message_id): + if self.token() is None: + raise Exception("User's Oauth2 token is None") + google = OAuth2Session(client_id, token=self.token()) + if self.token()['expires_at'] < time()+10: + google = self.refresh_token() + try: + r = google.get("https://www.googleapis.com/gmail/v1/users/me/messages/{}".format(message_id)) + except: + sleep(1) + return self.get_message(message_id) + return r.json() + + def remove_label(self, message_id): + if self.token() is None: + raise Exception("User's Oauth2 token is None") + google = OAuth2Session(client_id, token=self.token()) + if self.token()['expires_at'] < time()+10: + google = self.refresh_token() + params = {"removeLabelIds": ['CATEGORY_PROMOTIONS'], "addLabelIds": ['CATEGORY_PERSONAL']} + headers = {"Content-Type": "application/json"} + try: + r = google.post("https://www.googleapis.com/gmail/v1/users/me/messages/{}/modify".format(message_id), data=json.dumps(params), headers=headers) + except: + sleep(1) + r = google.post("https://www.googleapis.com/gmail/v1/users/me/messages/{}/modify".format(message_id), data=json.dumps(params), headers=headers) + return r.text + + def _validate_message(self, message_id): + print('new thread {}'.format(message_id)) + message = self.get_message(message_id) + for val in message['payload']['headers']: + if val['name'] == 'From': + sender = val['value'] + string = re.compile("<.+@(.+)>") + match = re.search(string, sender) + domain = match.group(1) + print(domain) + for whitelisted_domain in whitelist: + if whitelisted_domain in domain: # gracefully handle subdomains + print(domain, 'removed') + self.remove_label(row['id']) + + def clear_promo_folder(self): + promo_messages = self.get_messages() + for i, row in enumerate(promo_messages): + t = Thread(target=self._validate_message, args=(row['id'],)) + t.start() + return True + + def go(self): + self.clear_promo_folder() + return self.make_filter() + + def delete_filter(self): + if self.filter_id() is None: + raise Exception('Filter id not defined') + google = OAuth2Session(client_id, token=self.token()) + if self.token()['expires_at'] < time()+10: + google = self.refresh_token() + url = "https://www.googleapis.com/gmail/v1/users/me/settings/filters/{}".format(self.filter_id()) + print(url) + r = google.delete(url) + print(r.status_code) + if str(r.status_code)[0] == '2': + db = Db() + sql = 'UPDATE participant set filter_made = %s, filter_id = %s where email = %s;' + data = [False, None, self.email()] + db.query(sql, data) + self._filter_id = None + self._filter_made = False + return True + else: + print(r.text, r) + return False + + +if __name__=='__main__': + u = User() + + # get by email + #u.get_by_email('graydenshand@gmail.com') + #print(u.token()) + + # create + #u.create('graydenshand+test@gmail.com', 'Grayden Shand') + #print(u) + + # to json --> from json + #u.get_by_email('graydenshand@gmail.com') + #string = u.json() + #p = User(string) + #print(p) + + # user_info + #u.get_by_email('graydenshand@gmail.com') + #print(u.user_info()) + + # set_token + #token = {'access_token': 'ya29.ImCbB7-RCXlnETExBv945_682oO2VYjxCWT0lsnGCZ30FsoqZkcYH27GdkMEwXCfW7QgSKpzxf-XLgGbY-3HidpCV17dE3KTxti_EgbMZSaFLst1XzZmeIN0aym9J0tIb1c', 'expires_at': 1571326376.8499131, 'expires_in': 3600, 'id_token': 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjNkYjNlZDZiOTU3NGVlM2ZjZDlmMTQ5ZTU5ZmYwZWVmNGY5MzIxNTMiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20iLCJhenAiOiI1MTAyMzIyMzM3NTUtOXRldWdqNGRwODZnc2Y2NzU1MTFtZW52NWxubGJtYjkuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJhdWQiOiI1MTAyMzIyMzM3NTUtOXRldWdqNGRwODZnc2Y2NzU1MTFtZW52NWxubGJtYjkuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJzdWIiOiIxMDA0MTQ5OTU4NzM4NzI0MzUzOTQiLCJlbWFpbCI6ImdyYXlkZW5zaGFuZEBnbWFpbC5jb20iLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwiYXRfaGFzaCI6Im1BUGlVd0NvUUx4MDhuSmxwcU8wMXciLCJpYXQiOjE1NzEzMjI3NzYsImV4cCI6MTU3MTMyNjM3Nn0.zodpDP8UlpPlLPnNHRoOs6S47pPPCUYnETXbEuw64_kjkKvV-PWWhuNEBP-4kvZNT52712iIQ50VNJpeYbB2tD7Yd0R89eZwS8n7Z9YQPOjJ_o7sYJw57LSu7PEH3g3MquMLVry4Hh8Kd_uRuiqlFjjPJ5_x_HY-NshQTZf1R3bGTVnoEOHlyrJP0KQ8Xw3PiLJh_BnJo7xSKNI4ZB3Y2zTIxygzI57vOd0d-8GLOXNAObfvAtSJBIZznrF4MbWyxHSxzuVqXAeFTAHvgDQsiOHakXrWv4ZC77FVRMw7LGTUt6duhfyfub4pj6RtZIwVA6RVDm_1_m71wQOzr9FFTQ', 'refresh_token': '1/X2vsgWnYHG4Fzrfmqy_vK5azv8ZrCYqsdebWNjeqO1s', 'scope': ['https://www.googleapis.com/auth/userinfo.profile', 'openid', 'https://www.googleapis.com/auth/gmail.settings.basic', 'https://www.googleapis.com/auth/gmail.modify', 'https://www.googleapis.com/auth/userinfo.email'], 'token_type': 'Bearer'} + #u.get_by_email('graydenshand@gmail.com') + #u.set_token(token) + #u.get_by_email('graydenshand@gmail.com') + #print(u.token()) + + # make_filter + #u.get_by_email('graydenshand@gmail.com') + #u.make_filter() + #print(u.json()) + + #u.get_by_email('graydenshand@gmail.com') + #u.clear_promo_folder() + + u.get_by_email('graydenshand@gmail.com') + print(u.json()) + print(u.delete_filter()) + print(u.make_filter()) + + + + + + + +