update things

.gitignore

@@ -0,0 +1 @@
+auth.cfg

graphs.py

@@ -1,12 +1,13 @@
+from math import floor, ceil
+
 import plotly
 from plotly.graph_objs import Scatter, Layout, Figure
 
 
-def render_graph(followers_data):
-
+def render_graph(graph_data: dict):
     # prvni graf, tecky
     trace1 = Scatter(
-        x=followers_data[0],
+        x=graph_data['info_date_when'],
         y=[1, 2, 0, 2, 3, 0],
         mode='markers',
         marker={
@@ -22,7 +23,7 @@ def render_graph(followers_data):
 
     # druhý graf, čtverečky
     trace2 = Scatter(
-        x=followers_data[0],
+        x=graph_data['info_date_when'],
         y=[4, 3, 1, 2, 1, 2],
         mode='markers',
         marker={
@@ -39,16 +40,20 @@ def render_graph(followers_data):
 
     # treti graf, souvisla cara
     trace3 = Scatter(
-        x=followers_data[0],
-        y=followers_data[1],
+        x=graph_data['info_date_when'],
+        y=graph_data['info_followers_per_day'],
         mode='lines',
         name='followers',
     )
 
     data = [trace1, trace2, trace3]
 
+    min_followers = min(graph_data['info_followers_per_day'])
+    max_followers = max(graph_data['info_followers_per_day'])
+    min_followers = floor(min_followers / 100) * 100  # round to bottom 100
+    max_followers = ceil(max_followers / 100) * 100  # round to upper 100
     layout = Layout(
-        title='Double Y Axis Example',
+        title='Twitter Statistics for {}'.format(graph_data['user']['screen_name']),
         xaxis=dict(
             title='dates',
             zeroline=True,
@@ -59,16 +64,16 @@ def render_graph(followers_data):
             tickfont=dict(
                 color='rgb(148, 103, 189)'
             )
-         ),
+        ),
         yaxis=dict(
             title='followers',
-            range=[0, 400],
+            range=[min_followers, max_followers],
             zeroline=True,
-            showline = True,
+            showline=True,
             titlefont=dict(
                 color='rgb(148, 103, 189)'
             ),
-            tickfont = dict(
+            tickfont=dict(
                 color='rgb(148, 103, 189)'
             )
         ),
@@ -96,4 +101,4 @@ def render_graph(followers_data):
     fig = Figure(data=data, layout=layout)
     plotly.offline.plot(
         fig
-           )
+    )

project.py

@@ -1,8 +1,14 @@
+from datetime import datetime
+
 from graphs import render_graph
-from twitter import download_followers, count_followers, download_tweets
+from twitter import download_followers, count_followers, download_tweets, get_user
 
 if __name__ == '__main__':
-    # download_followers('yedpodtrzitko')
-    followers_date_and_sum = count_followers('yedpodtrzitko', '2017-06-30', '2017-07-06')
-    download_tweets('yedpodtrzitko')
-    render_graph(followers_date_and_sum)
+    user = get_user('yedpodtrzitko')
+    since = datetime(2017, 6, 30)
+    til = datetime(2017, 7, 6)
+    #download_followers(user)
+    graph_data = count_followers(user, since, til)
+    graph_data['user'] = user
+    download_tweets(user)
+    render_graph(graph_data)

twitter.py

@@ -2,39 +2,57 @@
 from datetime import datetime
 from twitter_wall import twitter_session
 import configparser
-from pprint import pprint
 
 
-# The program connects to the database. If not successful, raises the exception.
-# Then reads the API Key from the auth.cfg file and API Secret and opens the connection to twitter API.
-try:
-    conn = psycopg2.connect(dbname='cztwitter', user='jana', host='vanyli.net', password='!l3grac3ful', port=5433)
-    cur = conn.cursor()
-except:
-    print('I am unable to connect to the database')
+# from pprint import pprint
 
-config = configparser.ConfigParser()
-config.read('./auth.cfg')
-api_key = config['twitter']['key']
-api_secret = config['twitter']['secret']
 
-session = twitter_session(api_key, api_secret)
+def init_connections():
+    """
+    The program connects to the database. If not successful, raises the exception.
+    Then reads the API Key from the auth.cfg file and API Secret and opens the connection to twitter API.
+    """
+    config = configparser.ConfigParser()
+    config.read('./auth.cfg')
+
+    try:
+        conn = psycopg2.connect(
+            dbname=config['db']['name'],
+            user=config['db']['user'],
+            host=config['db']['host'],
+            password=config['db']['password'],
+            port=config['db']['port'],
+        )
+        cur = conn.cursor()
+    except Exception as e:
+        print(e)
+        raise Exception('I am unable to connect to the database')
+
+    api_key = config['twitter']['key']
+    api_secret = config['twitter']['secret']
+
+    session = twitter_session(api_key, api_secret)
+    return session, conn, cur
 
 
-def add_new_user(user_id, screen_name, do_check):
+session, conn, cur = init_connections()
+
+
+def add_new_user(user_id: int, screen_name: str, do_check: bool = False):
     """
     This function verifies, if the user is in the database. If not, this function inserts his id and nick to the table
     'twitter_user'.     It also inserts the information if the check is necessary.
     """
-    cur.execute('''SELECT id FROM twitter_user WHERE id = %s;''', (user_id,))        # 2nd parameter for execute() must be tuple
-    row = cur.fetchall()
-    if row == []:
+
+    cur.execute('''SELECT id FROM twitter_user WHERE id = %s;''', (user_id,))
+    row = cur.fetchone()
+    if not row:
         cur.execute('''INSERT INTO twitter_user(id, nick, do_check) VALUES (%s, %s, %s);''',
                     (user_id, screen_name, do_check))
         conn.commit()
 
 
-def add_followers(who, whom, followed_at):
+def add_followers(who: int, whom: int, followed_at: datetime):
     """
     The function inserts information about followers to the table 'follows' - id, id of the followed person and date.
     Each row must be unique. If the record already exists, it raises an exception and will not insert it again.
@@ -46,116 +64,122 @@ def add_followers(who, whom, followed_at):
         conn.rollback()
 
 
-def get_user_details(screen_name):
+def get_user_details(screen_name: str):
     """
     The function connects to api.twitter.com and returns the id of the user whose nick we typed as the parameter.
     """
     r = session.get('https://api.twitter.com/1.1/users/show.json',
                     params={'screen_name': screen_name}
                     )
-    id = r.json()['id']
-    return id
+    return r.json()
+
+
+def get_user(screen_name: str = None, user_id: int = None) -> dict:
+    user = cur.execute('SELECT * FROM twitter_user where nick = %s', (screen_name,))
+    if not user:
+        user = get_user_details(screen_name)
+        add_new_user(user['id'], screen_name, True)
+    else:
+        user = {
+            'id': user['id'],
+            'screen_name': user['nick'],
+        }
+    return user
 
 
-def download_followers(screen_name):
+def download_followers(user: dict):
     """
     This function downloads a dictionary from api.twitter.com. It contains a list of followers for the particular user
     listed in the function parameter and the 'cursor' information to allow paging. Each page has up to 200 entries.
     Using the 'add_new_user' and 'add_followers' functions, the information from all pages is inserted to the database.
     Do_check value is True for the followed users and Falls for followers.
     """
+    now = datetime.now()
     next_cursor = -1
+    print(user)
     while next_cursor != 0:
         r = session.get('https://api.twitter.com/1.1/followers/list.json',
-                        params={'screen_name': screen_name, 'cursor': next_cursor, 'count': 200}
+                        params={'screen_name': user['screen_name'], 'cursor': next_cursor, 'count': 20}
                         )
 
         followers = r.json()['users']
-        followed_id = get_user_details(screen_name)
-        add_new_user(followed_id, screen_name, True)
-        now = datetime.now()
         for follower in followers:
-            add_new_user(follower['id'], follower['screen_name'], False)
-            add_followers(follower['id'], followed_id, now)
+            add_new_user(follower['id'], follower['screen_name'])
+            add_followers(follower['id'], user['id'], now)
 
         next_cursor = r.json()['next_cursor']
-        print('Number of records:', len(followers), 'next_cursor:', next_cursor)
+        # print('Number of records:', len(followers), 'next_cursor:', next_cursor)
+        break
 
 
-def count_followers(screen_name, from_date, to_date):
+def count_followers(user: dict, from_date: datetime, to_date: datetime) -> dict:
     """
     The function counts all followers for the particular user and for each day within the specified period.
     It returns a dictionary that contains a list of dates and a list of numbers of followers for each day.
     """
-    first_day = datetime.strptime(from_date, '%Y-%m-%d')
-    last_day = datetime.strptime(to_date, '%Y-%m-%d')
-    followed_id = get_user_details(screen_name)
+
     cur.execute('''SELECT followed_at, COUNT(*) FROM follows 
         WHERE followed_at BETWEEN %s AND %s and whom = %s
-        GROUP BY followed_at ORDER BY followed_at;''', (first_day, last_day, followed_id))
+        GROUP BY followed_at ORDER BY followed_at;''', (from_date, to_date, user['id']))
+
     number_of_followers = cur.fetchall()
-    date_when = []
-    followers_per_day = []
-    for record in number_of_followers:
-        date_when.append(record[0])
-        followers_per_day.append(record[1])
+    date_when, followers_per_day = zip(*number_of_followers)
+
     followers_info = {
         'info_date_when': date_when,
-        'info_followers_per_day': followers_per_day
+        'info_followers_per_day': followers_per_day,
     }
     return followers_info
 
 
-def add_tweets_info(tweet_id, user_id, tweet_date, no_likes, no_retweets):
+def add_tweet_info(tweet: dict):
     """
     This function inserts information about tweets to the table 'tweets' - tweet_id, user_id, date,
     number of likes and number of retweets.
     """
-    cur.execute('''INSERT INTO tweets(tweet_id, user_id, tweet_date, no_likes, no_retweets) 
-        VALUES (%s, %s, %s, %s, %s);''', (tweet_id, user_id, tweet_date, no_likes, no_retweets))
-    conn.commit()
+    try:
+        cur.execute('''INSERT INTO tweets(tweet_id, user_id, tweet_date, no_likes, no_retweets) 
+            VALUES (%s, %s, %s, %s, %s);''', (
+            tweet['id'], tweet['user']['id'], tweet['created_at'], tweet['favorite_count'], tweet['retweet_count']
+        ))
+        conn.commit()
+    except:
+        conn.rollback()
 
 
-def update_tweets_info(no_likes, no_retweets, tweet_id):
+def update_tweet_info(tweet: dict):
     """
     This function updates number of likes and number of retweets in tweets table.
     """
     cur.execute('''UPDATE tweets SET(no_likes, no_retweets) = (%s, %s) WHERE tweet_id = (%s);''',
-                (no_likes, no_retweets, tweet_id))
+                (tweet['favorite_count'], tweet['retweet_count'], tweet['id']))
     conn.commit()
 
 
-def download_tweets(screen_name):
+def download_tweets(user: dict):
     """
     This function downloads a list of dictionaries from api.twitter.com. It contains information about
     tweets and twitter user. Using the add_tweet_info function, the information is inserted to the database.
     It inserts only original tweets (no retweets).
     """
     r = session.get('https://api.twitter.com/1.1/statuses/user_timeline.json',
-                    params={'screen_name': screen_name, 'count': 200, 'include_rts': False}
-                    )
+                    params={'screen_name': user['screen_name'], 'count': 200, 'include_rts': False})
     tweets = r.json()
     for tweet in tweets:
         cur.execute('''SELECT tweet_id FROM tweets WHERE tweet_id = %s;''', (tweet['id'],))
         row = cur.fetchone()
-        if row is None:
-            add_tweets_info(tweet['id'], tweet['user']['id'], tweet['created_at'], tweet['favorite_count'],
-                            tweet['retweet_count'])
-            conn.commit()
+        if not row:
+            add_tweet_info(tweet)
         else:
-            update_tweets_info(tweet['favorite_count'], tweet['retweet_count'], tweet['id'])
-            conn.commit()
+            update_tweet_info(tweet)
 
 
-def count_tweets(screen_name, from_date, to_date):
+def count_tweets(screen_name: str, from_date: datetime, to_date: datetime):
     """
     This function counts all tweets for the particular user and for each day within the specified period.
     """
-    first_day = datetime.strptime(from_date, '%Y-%m-%d')
-    last_day = datetime.strptime(to_date, '%Y-%m-%d')
     followed_id = get_user_details(screen_name)
     cur.execute('''SELECT tweet_date, COUNT(*) FROM tweets
         WHERE tweet_date BETWEEN %s AND %s AND user_id = %s
-        GROUP BY tweet_date ORDER BY tweet_date'''), (first_day, last_day, followed_id)
-    number_of_tweets = cur.fetchall
-
+        GROUP BY tweet_date ORDER BY tweet_date'''), (from_date, to_date, followed_id)
+    number_of_tweets = cur.fetchall()