Merge pull request #3 from sziyan/database

v1. 1 release

app/__init__.py

@@ -0,0 +1,12 @@
+from config import Config
+import sqlalchemy as db
+from sqlalchemy.orm import sessionmaker
+
+engine = db.create_engine(Config.SQLALCHEMY_DATABASE_URI)
+connection = engine.connect()
+Session = sessionmaker()
+Session.configure(bind=engine)
+session = Session()
+
+
+from app import models

app/models.py

@@ -0,0 +1,30 @@
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column,Integer,String, Float
+Base = declarative_base()
+
+class Games(Base):
+    __tablename__ = 'games'
+    id = Column(Integer, primary_key=True)
+    title = Column(String, nullable=False)
+    link = Column(String, nullable=False)
+    count = Column(Integer, nullable=True)
+    rating = Column(Float, nullable=True)
+    price = Column(String, nullable=True)
+    category = Column(String, nullable=True)
+    developer = Column(String, nullable=True)
+    iap_range = Column(String, nullable=True)
+    month = Column(String, nullable=True)
+
+
+class Apps(Base):
+    __tablename__ = 'apps'
+    id = Column(Integer, primary_key=True)
+    title = Column(String, nullable=False)
+    link = Column(String, nullable=False)
+    count = Column(Integer, nullable=True)
+    rating = Column(Float, nullable=True)
+    price = Column(String, nullable=True)
+    category = Column(String, nullable=True)
+    developer = Column(String, nullable=True)
+    iap_range = Column(String, nullable=True)
+    month = Column(String, nullable=True)

config_sample.py

@@ -0,0 +1,15 @@
+import os
+basedir = os.path.abspath(os.path.dirname(__file__))
+
+class Config:
+    client_id =  'reddit client id '#reddit client id
+    client_secret =  'reddit client secret' #reddit client secret
+    username = 'reddit username'
+    password = 'reddit password'
+    user_agent = 'user_agent'
+    subreddit = ['google_play'] #subreddit to monitor
+    max_apps = 15 #max number of apps to list in 1 single comment
+    SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or 'sqlite:///' + os.path.join(basedir, 'mydb.db') #sqlalchemy URI
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    token = 'token' #telegram bot token
+    chat_id = 'chat_id' #telegram chat id

create_table.py

@@ -0,0 +1,5 @@
+from app.models import Base
+from app import engine
+
+print('Database updated')
+Base.metadata.create_all(engine)

manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(repository='migration', url='sqlite:///gplay.db', debug='False')

migration.bat

@@ -0,0 +1,15 @@
+@echo off
+echo "Make sure in virtualenv folder."
+pause
+
+echo "Create migration folder"
+
+migrate create migration "Project"
+
+set /p db=Enter database filename(with .db):
+
+echo "Setting up version control."
+python migration/manage.py version_control sqlite:///%db% migration
+
+echo "Setting path."
+migrate manage manage.py --repository=migration --url=sqlite:///%db%

migration/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

migration/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

migration/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Project
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

monthly.log

@@ -0,0 +1,7 @@
+09-Jan-20 12:09:40 PM INFO - Monthly top games posted: https://www.reddit.com/r/sziyan_testing/comments/em4dta/top_10_linkme_games_of_the_month/
+09-Jan-20 12:11:25 PM INFO - Monthly top games posted: https://www.reddit.com/r/sziyan_testing/comments/em4eiv/top_10_linkme_games_of_the_month/
+09-Jan-20 12:12:35 PM INFO - Monthly top games posted: https://www.reddit.com/r/sziyan_testing/comments/em4f1o/top_10_linkme_games_of_the_month/
+09-Jan-20 12:13:15 PM INFO - Monthly top games posted: https://www.reddit.com/r/sziyan_testing/comments/em4fdp/top_10_linkme_games_of_the_month/
+09-Jan-20 02:15:38 PM INFO - Monthly top games posted: https://www.reddit.com/r/sziyan_testing/comments/em5pf6/top_1_linkme_games_of_the_month/
+09-Jan-20 02:20:07 PM INFO - Monthly top games posted: https://www.reddit.com/r/sziyan_testing/comments/em5qvy/top_1_linkme_games_of_the_month/
+09-Jan-20 03:59:27 PM INFO - Monthly top games posted: https://www.reddit.com/r/sziyan_testing/comments/em6ngc/top_5_linkme_games_of_the_month/

playstore.py

@@ -9,6 +9,9 @@
 import play_scraper as play
 from config import Config
 import requests
+from app.models import Games,Apps
+from app import session
+import datetime
 
 logging.basicConfig(level=logging.INFO, filename='output.log', filemode='a', format='%(asctime)s %(levelname)s - %(message)s', datefmt='%d-%b-%y %I:%M:%S %p')
 logging.info("Bot started successfully")
@@ -18,6 +21,62 @@
 logging.info("Watching /r/{} ...".format((",/r/").join(Config.subreddit)))
 logging.info("Waiting for comments...")
 
+
+def update_db(subreddit, result):
+    title = result['title']
+    link = result['url']
+    rating = result['score']
+    if result['free']is True:
+        price = 'Free'
+    else:
+        price = result['price']
+    if result['iap'] is True:
+        price += ' with IAP'
+    category_list_types = result['category']
+    category_list = []
+    for i in category_list_types:
+        if '_' in i:
+            in_list = i.split('_')
+            category_list.append(in_list[1].capitalize())
+        else:
+            category_list.append(i.capitalize())
+    category = ", ".join(category_list)
+    developer = result['developer']
+    if result['iap_range'] is None:
+        iap_range = 'NA'
+    else:
+        iap_range = " - ".join(result['iap_range'])
+    now = datetime.datetime.now()
+    month = now.strftime('%b %y')
+
+    if subreddit == 'AndroidGaming' or subreddit == 'sziyan_testing':
+        check_db = session.query(Games).filter_by(title = title).first()
+        if check_db is None: #game does not exist
+            game = Games(title=title, link=link,rating=rating,price=price,category=category,count=1,developer=developer,iap_range=iap_range, month=month)
+            session.add(game)
+            session.commit()
+            logging.info('New game {} added to database.'.format(title))
+            return 1
+        else: #game already exist
+            check_db.count += 1
+            session.commit()
+            return 1
+    elif subreddit == 'AndroidApps':
+        check_db = session.query(Apps).filter_by(title=title).first()
+        if check_db is None:
+            app = Apps(title=title, link=link,rating=rating,price=price,category=category,count=1,developer=developer,iap_range=iap_range, month=month)
+            session.add(app)
+            session.commit()
+            logging.info("New app {} added to database.".format(title))
+            return 1
+        else:
+            check_db.count+=1
+            session.commit()
+            return 1
+    else:
+        logging.warning("Invalid subreddit. This should no happen unless testing!")
+        return 0
+
 def sendtelegram(message):
     token = Config.token
     chat_id = Config.chat_id
@@ -63,6 +122,7 @@ def get_all_app_requests(linkme_requests):
             app_count = get_no_apps(link_me_requests)
             app_list = get_all_app_requests(link_me_requests)
             count = 1
+            result_list = []
             if app_count > 0:
                 logging.info("{} is searching for {} app(s) in /r/{}: {}".format(comments.author.name, app_count,comments.subreddit.display_name, ",".join(app_list)))
                 print("{} is searching for {} app(s): {}".format(comments.author.name, app_count, ",".join(app_list)))
@@ -99,6 +159,7 @@ def get_all_app_requests(linkme_requests):
                         msg = "**[{}]({})** | {}  | {} | {} downloads | [Search manually]({}) \n\n> {}".format(title,url,score,price,installs,search_manual,desc_output)
                     else:
                         msg = "[{}]({}) - {} - {} - [Search manually]({}) \n\n".format(title,url,score,price,search_manual)
+                    result_list.append(result)
                     count+=1
                     message+=msg
                 if message != "":
@@ -110,6 +171,10 @@ def get_all_app_requests(linkme_requests):
                 else:
                     logging.info("{} searched for an empty game.".format(comments.author.name))
                     continue
+                for result in result_list:
+                    update_db_result = update_db(comments.subreddit.display_name, result)
+                    if update_db_result is not 1:
+                        logging.warning('Updating of database failed.')
             else:
                 continue
         except praw.exceptions.APIException as api_exception:

post_monthly.py

@@ -0,0 +1,45 @@
+import logging
+import praw
+from config import Config
+from app.models import Games,Apps
+from app import session
+import datetime
+
+logging.basicConfig(level=logging.INFO, filename='monthly.log', filemode='a', format='%(asctime)s %(levelname)s - %(message)s', datefmt='%d-%b-%y %I:%M:%S %p')
+current_month = datetime.datetime.now()
+month = current_month.strftime('%b %y')
+month_game = session.query(Games).filter_by(month=month)
+games = month_game.order_by(Games.count.desc())
+#games = session.query(Games).order_by(Games.count.desc())
+
+number_of_games = session.query(Games).count()
+msg = ""
+top_games = []
+reddit = praw.Reddit(client_id=Config.client_id, client_secret=Config.client_secret, username=Config.username, password=Config.password, user_agent=Config.user_agent)
+#android_gaming = reddit.subreddit('AndroidGaming')
+subreddit = reddit.subreddit('sziyan_testing')
+
+if number_of_games > Config.max_monthly:
+    for i in range(0,Config.max_monthly):
+        top_games.append(games[i])
+else:
+    for i in range(0,number_of_games):
+        top_games.append(games[i])
+
+##introduction message
+msg += 'Hello /r/AndroidGaming! \n\n' \
+       'This is a list of top {} games that was found using linkme. \n\n' \
+       'Any feedbacks are greatly appreciated! \n\n\n\n'.format(Config.max_monthly)
+
+##table header
+msg += '| Count | Title | URL | Ratings | Price | Category | Developer | IAP |\n'
+msg += '| :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: |\n' # seperate header and body
+
+for i in top_games:
+    msg += '| {} | {} | [Google Play]({}) | {} | {} | {} | {} | {} |\n'.format(i.count, i.title, i.link, i.rating, i.price, i.category, i.developer, i.iap_range)
+
+msg += 'This post is automated at the end of every month.'
+
+submission = subreddit.submit(title='Top {} linkme games of the month!'.format(Config.max_monthly), send_replies=False, selftext=msg)
+logging.info('Monthly top games posted: {}'.format(submission.url))
+

requirements.txt

@@ -2,11 +2,14 @@ beautifulsoup4==4.8.2
 certifi==2019.11.28
 cffi==1.13.2
 chardet==3.0.4
+config==0.4.2
 cryptography==2.8
+decorator==4.4.1
 html2text==2019.9.26
 idna==2.8
 lxml==4.4.2
 Markdown==3.1.1
+pbr==5.4.4
 play-scraper==0.6.0
 praw==6.4.0
 prawcore==1.0.1
@@ -16,6 +19,10 @@ requests==2.22.0
 requests-futures==1.0.0
 six==1.13.0
 soupsieve==1.9.5
+SQLAlchemy==1.3.12
+sqlalchemy-migrate==0.13.0
+sqlparse==0.3.0
+Tempita==0.5.2
 update-checker==0.16
 urllib3==1.25.7
 websocket-client==0.57.0

scripts/migration link.txt

@@ -0,0 +1 @@
+https://sqlalchemy-migrate.readthedocs.io/en/latest/versioning.html#project-setup

test.py

@@ -1,66 +1,5 @@
-# import play_scraper as play
-# import praw
-# from config import Config
-#
-# reddit = praw.Reddit(client_id=Config.client_id, client_secret=Config.client_secret, username=Config.username, password=Config.password, user_agent=Config.user_agent)
-# subreddit = reddit.subreddit('sziyan_testing')
-#
-# for comments in subreddit.comments(limit=100):
-#     print(comments.author.name)
-#     comments.delete()
-#
-# print("deleted")
-#
-# for comments in subreddit.comments():
-#     print(comments.author)
-
-#!/home/sziyan/bots/reddit/gplay/venv/bin python3
-
-import logging
-import re
-import praw
-from praw import exceptions
-import markdown
-import html2text
 import play_scraper as play
-from config import Config
-
-logging.basicConfig(level=logging.INFO, filename='output.log', filemode='a', format='%(asctime)s %(levelname)s - %(message)s', datefmt='%d-%b-%y %I:%M:%S %p')
-logging.info("Bot started successfully")
-
-reddit = praw.Reddit(client_id=Config.client_id, client_secret=Config.client_secret, username=Config.username, password=Config.password, user_agent=Config.user_agent)
-subreddit = reddit.subreddit("+".join(Config.subreddit))
-logging.info("Watching /r/{} ...".format((",/r/").join(Config.subreddit)))
-logging.info("Waiting for comments...")
-
-
-link_me_regex = re.compile("\\blink[\s]*me[\s]*:[\s]*(.*?)(?:\.|;|$)", re.M | re.I)
-
-def get_clean_comment(text):
-    text_maker = html2text.HTML2Text()
-    text_maker.ignore_emphasis = True
-    text_maker.IGNORE_ANCHORS = True
-    text_maker.IGNORE_IMAGES = True
-    htmltext = markdown.markdown(text)
-    output = text_maker.handle(htmltext)
-    return output
-
-def get_no_apps(linkme_requests):
-    app_count = 0
-    for rq in linkme_requests:
-        for item in rq.split(','):
-            app_count += 1
-    return app_count
-
-def get_all_app_requests(linkme_requests):
-    apps_list = []
-    for rq in linkme_requests:
-        for item in rq.split(","):
-            apps_list.append(item)
-    return apps_list
-
-for comments in subreddit.stream.comments(skip_existing=True):
-    print(comments)
-
-
 
+game = 'clash of clans'
+result = play.search(game,detailed=True,page=1)[0]
+print(result['iap_range'])