twitter_post_scraper.py

import requests
from bs4 import BeautifulSoup
import re

re_text = r'\:|\.|\!|(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b|(.twitter.com\/)\w*|\&'
re_text_1 = r'(pictwittercom)\/\w*'

def tweeter_scrapper():
    list_of_dirty_tweets = []
    clear_list_of_tweets = []
    base_tweeter_url = 'https://twitter.com/{}'

    tweeter_id = input()

    response = requests.get(base_tweeter_url.format(tweeter_id))
    soup = BeautifulSoup(response.content , 'lxml')
    all_tweets = soup.find_all('div',{'class':'tweet'})

    for tweet in all_tweets:
        content = tweet.find('div',{'class':'content'})
        message = content.find('div',{'class':'js-tweet-text-container'}).text.replace("\n"," ").strip()
        list_of_dirty_tweets.append(message)
    for dirty_tweet in list_of_dirty_tweets:
        dirty_tweet = re.sub(re_text, '', dirty_tweet, flags=re.MULTILINE)
        dirty_tweet = re.sub(re_text_1, '', dirty_tweet, flags=re.MULTILINE)
        dirty_tweet = dirty_tweet.replace(u'\xa0…', u'')
        dirty_tweet = dirty_tweet.replace(u'\xa0', u'')
        dirty_tweet = dirty_tweet.replace(u'\u200c', u'')
        clear_list_of_tweets.append(dirty_tweet)
    print(clear_list_of_tweets)



if __name__ == "__main__":
    tweeter_scrapper()