Merge pull request avinashkranjan#903 from RohiniRG/RohiniRG-twitterb

Twitter Scraper using snscrape

Author: avinashkranjan

Twitter_Scraper_without_API/README.md

@@ -0,0 +1,41 @@
+# Tweet hashtag based scraper without Twitter API
+
+- Here, we make use of snscrape to scrape tweets associated with a particular hashtag. Snscrape is a python library that scrapes twitter without the use of API keys.
+
+- We have 2 scripts associated with this project one to fetch tweets with snscrape and store it in the database (we use SQLite3), and the other script displays the  tweets from the database.
+
+- Using snscrape, we are storing the hashtag, the tweet content, user id, as well as the URL of the tweets in the database.
+
+## Requirements
+
+Packages associated can be installed as:
+
+```sh
+  $ pip install -r requirements.txt
+```
+
+## Running the script
+
+For running the script which fetches tweets and other info associated with the hashtag and storing in the database:
+```sh
+  $ python fetch_hashtags.py
+```
+
+For running the script to display the tweet info stored in the database:
+```sh
+  $ python display_hashtags.py
+```
+
+## Working
+
+```fetch_hashtags.py``` will work as follows:
+
+![image](https://imgur.com/8YFK4OV.png)
+
+```display_hashtags.py``` will work as follows:
+
+![image](https://i.imgur.com/1uNEEMw.png)
+
+## Author
+
+[Rohini Rao](https://github.com/RohiniRG)

Twitter_Scraper_without_API/display_hashtags.py

@@ -0,0 +1,50 @@
+import sqlite3
+import os
+
+
+def sql_connection():
+    """
+    Establishes a connection to the SQL file database
+    :return connection object:
+    """
+    path = os.path.abspath('./Twitter_Scraper_without_API/TwitterDatabase.db')
+    con = sqlite3.connect(path)
+    return con
+
+
+def sql_fetcher(con):
+    """
+    Fetches all the tweets with the given hashtag from our database
+    :param con:
+    :return:
+    """
+    hashtag = input("\nEnter hashtag to search: #")
+    hashtag = '#' + hashtag
+    count = 0
+    cur = con.cursor()
+    cur.execute('SELECT * FROM tweets')  # SQL search query
+    rows = cur.fetchall()
+
+    for r in rows:
+        if hashtag in r:
+            count += 1
+            print(f'USERNAME: {r[1]}\nTWEET CONTENT: {r[2]}\nURL: {r[3]}\n')
+
+    if count:
+        print(f'{count} tweets fetched from database')
+    else:
+        print('No tweets available for this hashtag')
+
+
+con = sql_connection()
+
+while 1:
+    sql_fetcher(con)
+
+    ans = input('Press (y) to continue or any other key to exit: ').lower()
+    if ans == 'y':
+        continue
+    else:
+        print('Exiting..')
+        break
+

Twitter_Scraper_without_API/fetch_hashtags.py

@@ -0,0 +1,66 @@
+import snscrape.modules.twitter as sntweets
+import sqlite3
+
+
+def sql_connection():
+    """
+    Establishes a connection to the SQL file database
+    :return connection object:
+    """
+    con = sqlite3.connect('./Twitter_Scraper_without_API/TwitterDatabase.db')
+    return con
+
+
+def sql_table(con):
+    """
+    Creates a table in the database (if it does not exist already)
+    to store the tweet info
+    :param con:
+    :return:
+    """
+    cur = con.cursor()
+    cur.execute("CREATE TABLE IF NOT EXISTS tweets(HASHTAG text, USERNAME text,"
+                " CONTENT text, URL text)")
+    con.commit()
+
+
+def sql_insert_table(con, entities):
+    """
+    Inserts the desired data into the table to store tweet info
+    :param con:
+    :param entities:
+    :return:
+    """
+    cur = con.cursor()
+    cur.execute('INSERT INTO tweets(HASHTAG, USERNAME, CONTENT, '
+                'URL) VALUES(?, ?, ?, ?)', entities)
+    con.commit()
+
+
+con = sql_connection()
+sql_table(con)
+
+while 1:
+    tag = input('\n\nEnter a hashtag: #')
+    max_count = int(input('Enter maximum number of tweets to be listed: '))
+
+    count = 0
+    # snscrape uses the given string of hashtag to find the desired amount of
+    # tweets and associated info
+    for i in sntweets.TwitterSearchScraper('#' + tag).get_items():
+        count += 1
+        entities = ('#'+tag, i.username, i.content, i.url)
+        sql_insert_table(con, entities)
+
+        if count == max_count:
+            break
+
+    print('Done!')
+
+    ans = input('Press (y) to continue or any other key to exit: ').lower()
+    if ans == 'y':
+        continue
+    else:
+        print('Exiting..')
+        break
+

Twitter_Scraper_without_API/requirements.txt

@@ -0,0 +1,10 @@
+beautifulsoup4==4.9.3
+certifi==2020.12.5
+chardet==4.0.0
+idna==2.10
+lxml==4.6.2
+PySocks==1.7.1
+requests==2.25.1
+snscrape==0.3.4
+soupsieve==2.2
+urllib3==1.26.4