goodreadsScrape.py

# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import urllib.request as req
from bs4 import BeautifulSoup
import os
import time
import sys
import json


#converts list to json and writes data in file
def writeToJSON(data):
    outputfilename = "temp.json"
    with open(outputfilename, 'w') as f:
        d = json.dumps(data)
        f.write(d)

#gets data response for the request
def getResponse(reqURL):
    response = req.urlopen(reqURL)
    data = response.read()
    return data

#Tag list is generated by scraping 34 pages from goodreads.com
tags_list=['love', 'life', 'inspirational', 'humor', 'philosophy', 'truth', 'god', 'wisdom', 'inspirational-quotes', 'happiness', 'romance', 'hope', 'death', 'quotes', 'poetry', 'faith', 'writing', 'inspiration', 'religion', 'knowledge', 'success', 'education', 'relationships', 'motivational', 'life-lessons', 'time', 'science', 'funny', 'books', 'spiritualityfear', 'purpose', 'motivation', 'love-quotes', 'friendship', 'war', 'people', 'dreams', 'spiritual', 'freedom', 'humour', 'life-quotes', 'soul', 'politics', 'women', 'change', 'beauty', 'christianity', 'art', 'leadership', 'reading', 'debasish-mridha', 'intelligence', 'history', 'humanity', 'nature', 'work', 'peace', 'pain', 'self-helpmarriage', 'music', 'motivational-quotes', 'money', 'heart', 'fantasy', 'christian', 'mind', 'family', 'power', 'world', 'courage', 'quote', 'passion', 'reality', 'positive-thinking', 'sex', 'joy', 'wisdom-quotes', 'spiritual-quotes', 'children', 'feminism', 'fiction', 'society', 'strength', 'psychology', 'living', 'self-improvement', 'destiny', 'success-quotescreativity', 'belief', 'future', 'depression', 'literature', 'words', 'prayer', 'trust', 'meditation', 'life-and-living', 'learning', 'relationship', 'light', 'inspiring', 'forgiveness', 'grief', 'loss', 'sadness', 'life-philosophy', 'business', 'imagination', 'friends', 'men', 'self', 'food-for-thought', 'inspirational-life', 'magic', 'dream', 'compassion', 'jesusjesus', 'paranormal-romance', 'kindness', 'thoughts', 'travel', 'loneliness', 'wealth', 'spirit', 'debasish-mridha-m-d', 'growth', 'wise-words', 'personal-development', 'israelmore-ayivor', 'goals', 'suffering', 'meaning', 'past', 'self-esteem', 'desire', 'calling', 'confidence', 'failure', 'advice', 'self-love', 'healing', 'mystery', 'debasish-mridha-md', 'suicide', 'personal-growth', 'young-adultrelationship-quotes', 'darkness', 'adventure', 'humorous', 'writers', 'poverty', 'grace', 'fate', 'evil', 'horror', 'consciousness', 'contemporary-romance', 'understanding', 'billy-graham', 'action', 'science-fiction', 'poem', 'experience', 'self-awareness', 'language', 'believe', 'sad', 'ignorance', 'anger', 'culture', 'mind-power', 'man', 'feelings', 'thinking', 'hatememories', 'morality', 'beautiful', 'greatness', 'human', 'honesty', 'food', 'silence', 'mind-body-spirit', 'parenting', 'lies', 'positive-thoughts', 'universe', 'identity', 'bible', 'lovers', 'christian-life', 'heartbreak', 'inspirational-attitude', 'edmond-mbiaka', 'spiritual-growth', 'self-realization', 'journey', 'government', 'lailah-gifty-akita-affirmations', 'choices', 'character', 'emotions', 'health', 'human-naturewoman', 'hurt', 'atheism', 'value', 'justice', 'mindfulness', 'sin', 'good', 'humility', 'evolution', 'awareness', 'positive', 'perspective', 'gratitude', 'stories', 'poetry-quotes', 'book', 'philosophy-of-life', 'attitude', 'home', 'spiritual-wisdom', 'optimism', 'youth', 'happy', 'heaven', 'encouragement', 'paranormal', 'positive-attitude', 'amor', 'choicerespect', 'communication', 'acceptance', 'story', 'pride', 'writer', 'focus', 'novel', 'self-motivation', 'author', 'determination', 'unconditional-love', 'islam', 'romantic', 'ya', 'positive-quotes', 'opportunity', 'existence', 'greatness-quotes', 'gift', 'reason', 'sacred', 'vampires', 'richelle', 'richelle-goodrich', 'childhood', 'live', 'thought', 'america', 'love-storylove-story', 'crime', 'earth', 'communication-skills', 'solitude', 'patience', 'positive-outlook', 'life-quotes-and-sayings', 'church', 'violence', 'worship', 'inspire', 'smile', 'ego', 'life-lessons-quotes', 'inspirational-quote', '인터넷경마', 'buddhism', 'stars', 'values', 'anxiety', 'perseverance', '온라인경륜', 'enlightenment', 'kiss', 'technology', '사설경마사이트', '마권판매사이트', 'perception', 'servicevision', 'survival', 'dark', 'sacrifice', 'self-confidence', 'true-love', 'power-of-words', 'law', 'racism', 'teaching', 'democracy', 'job', 'life-lesson-quotes', 'sleep', 'creation', 'faith-quotes', 'employment', 'responsibility', 'thriller', 'storytelling', 'mistakes', 'matshona-dhliwayo-quotes', 'great', 'happiness-quotes', 'positivity', 'jesus-christ', '온라인경마', 'memoir', 'regret', 'sorrowyou', 'capitalism', 'theology', 'bravery', 'master', 'alone', 'adversity', 'hell', 'artist', 'wise-quotes', 'empowerment', 'achievement', 'ideas', 'dating', 'poems', 'growing-up', 'vampire', 'sufis', '办理国外文凭', 'murat-ildan-sözleri', 'ethics', '온라인경마사이트', 'comedy', 'path', 'christ', 'revolution', 'soul-searching', 'animals', '온라인경정', 'discipleshipintegrity', 'positive-motivation', 'mother', 'power-of-thoughts', 'murder', 'humans', 'control', 'equality', 'positive-mindset', 'despair', 'mastery', 'productivity', 'spiritual-development', 'sexuality', 'mridha', 'school', 'dreams-quotes', '일본경마사이트', 'wisdom-of-lailah-gifty-akita', 'christian-living', 'love-quote', 'age', 'knowledge-of-self', 'progress', 'abuse', '인터넷경마사이트', 'night', 'debasish', 'mental-illness', 'economicsteachings', 'friendship-quotes', 'sage', 'words-of-wisdom', 'quotes-by-susan-c-young', 'power-of-love', 'emotion', 'sarcasm', 'lost', 'secrets', 'parents', 'blessing', 'love-quotes-and-sayings', 'turkish-literature', 'leaders', 'problems', 'jobless', 'law-of-attraction', 'historical-romance', 'romance-novels', 'quotes-to-live-by', 'zen', 'ambition', 'motherhood', 'satire', 'mehmet-murat-ildan', 'opportunities', 'life-quote', 'suicidal-thoughts', 'entrepreneurshipknowing-oneself', 'philosophical', 'christmas', 'personality', 'spiritualism', 'writing-life', 'body', 'erotica', 'sage-advice', 'energy', 'life-quotations', 'knowledge-education', 'vida', 'goal-setting', 'doubt', 'suspense', 'writing-process', 'irony', 'sacred-wisdom', 'erotic-romance', 'masters', 'humorous-quotes', 'challenges', 'self-improvement-book', 'danger', 'knowledge-teaching', 'religious', 'romantic-suspense', 'unconditional-acceptance', 'self-worthself-worth', 'mankind', 'waste', 'write', 'sacred-teachings', 'lust', 'powerful-story', 'revenge', 'decisions', 'curiosity', 'innovation', 'empathy', 'life-changing', 'master-of-love', 'turkish-writers', 'angels', 'master-key', 'community', 'self-esteem-self-help-book', 'leader', 'kindlehighlight', 'death-and-dying', 'wonder', 'fight', 'perfection', 'writers-on-writing', 'brainy-quotes', 'eyes', 'madness', 'transformationtransformation', 'true', 'turkish-quotes', 'dying', '경마예상', 'actions', 'goodness', 'individuality', 'cats', 'hatred', 'love-hurts', 'wise', 'goal', 'unity', 'salvation', 'guilt', 'mehmet-murat-ildan-quotes', 'letting-go', 'addiction', 'richelle-e-goodrich', 'help', 'greed', 'historical-fiction', 'drama', 'quote-of-the-day', 'aging', 'race', 'finance', 'people-quotes', 'betrayalstruggle', 'pleasure', 'honor', 'brain', 'life-quotes-inspirational-quotes', 'mental-health', 'mindset', 'life-experience', 'stephen-richards', 'sexy', 'tragedy', 'learn', 'criticism', 'girls', 'mission', 'uplifting', 'sweet', 'writing-advice', 'dead', 'space', 'wise-sayings', 'existentialism', 'conversion', 'meaning-of-life', 'expectations', 'will', 'slavery', 'destruction', 'drugs', 'reflectionmehmet-murat-ildan-quotations', 'young-adult-fiction', 'hard-work', 'kingdom', 'attitude-quotes', 'mercy', 'logic', 'tears', 'intuition', 'dogs', 'özlü-sözler', 'life-lessons-quotations', 'product', 'talent', 'devil', 'environment', 'illusion', 'excellence', 'nostalgia', 'comfort', 'ildan-wise-sayings', 'friend', 'ildan-sözleri', 'turkish-authors', 'beliefs', 'potential', 'dragons', 'genius', 'conscience', 'özdeyişlergay', 'dance', 'turkish-playwrights', 'authors', 'gender', 'loyalty', 'supernatural', 'virtue', 'divorce', 'new-thought', 'turkish-wisdom-words', 'growth-quotes', 'motivational-inspirational', 'poet', 'dreaming', 'turkish-quotations', 'urban-fantasy', 'produce', 'gratitude-quotes', 'water', 'read', 'ildan-wisdom', 'ildan-words', 'türk-yazar-sözleri', 'insanity', 'laughter', 'stupidity', 'jealousy', 'think', 'childnew-adult', 'witches', 'media', 'moon', 'wishes', 'potential-quotes', 'free', 'fashion', 'principles', 'joblessness', 'ghosts', 'gods', 'thinking-quotes', 'truth-of-life', 'conflict', 'fearless', 'eternity', 'father', 'self-discovery', 'rain', 'influence', 'miracles', 'strategy', 'sun', 'business-quotes', 'sufism', 'right', 'social-media', 'convert', 'cutematerialism', 'shame', 'never-give-up', 'demons', 'new-age', 'giving', 'ocean', 'military', 'influence-quotes', 'independence', 'türk-yazarlar', 'marketing', 'relationships-quotes', 'karma', 'educational-philosophy', 'lie', 'longing', 'hero', 'fun', 'recovery', 'sky', 'judgement', 'photography', 'loving', 'shakespeare', 'türk-yazar-özdeyişleri', 'communism', 'motivational-speaker', 'darbımesel', 'kelamıkibarkelamıkibar', 'civilization', 'wisdom-quote', 'life-lesson', 'mehmet-murat-ildan-özlü-sözleri', 'worry', 'crazy', 'corruption', 'reality-of-life', 'desires', 'book-quotes', 'moving-on', 'lgbt', 'trauma', 'luck', 'want', 'inner-strength', 'wisdom-quotations', 'political', 'risk', 'pray', 'chaos', 'feeling', 'selfishness', 'harmony', 'sea', 'blessings', 'appreciation', 'patriotism', 'wealth-creationwealth-creation', 'winter', '경마사이트', 'fulfillment', 'need', 'mythology', 'simplicity', 'prejudice', 'life-and-death', 'victory', 'poets', 'divine', 'management', 'internet', 'africa', 'lonely', 'socialism', 'successful-living', 'metaphor', 'kids', 'physics', 'motivational-quotes-for-women', 'motivational-quote', 'appreciation-quotes', 'library', 'thought-provoking', 'medicine', 'manifestation', 'persistence', 'speechpersistence', 'inspirational-living', 'disappointment', 'care', 'artists', 'cinta', 'dream-big', 'crying', 'hypocrisy', 'wise-quotations', 'oscar-wilde', 'sports', 'questions', 'misery', 'flowers', 'inspirational-quotes-about-life', 'mysticism', 'listening', 'weakness', 'stress', 'broken', 'day', 'contemporary', 'fame', 'emotional', 'falling-in-love', 'spirituality-quotes', 'moment', 'percy-jackson', 'strongquotes-about-life', 'lover', 'discovery', 'birth', 'deep', 'inner-peace', 'poor', 'coming-of-age', 'immortality', 'being', 'vision-quotes', 'self-belief', 'maturity', 'contentment', 'good-and-evil', 'arrogance', 'alcohol', 'questions-quotes', 'fairy-tales', 'deception', 'mathematics', 'career', 'secret', 'person', 'discipline', 'writing-quotes', 'blood', 'rich', 'yoga', 'abundance-quoteswrong', 'heartache', 'prosperity', 'sci-fi', 'commitment', 'drinking', 'battle', 'authenticity', 'sage-quotes', 'forever', 'humanism', 'lyrics', 'real', 'harry-potter', 'girl', 'believe-in-yourself', 'india', 'waiting', 'heroes', 'funny-quotes', 'die', 'improvement', 'biology', 'holy-spirit', 'lessons', 'winning', 'connection', 'new-york-city', 'sufismo', 'summersummer', 'sexism', 'attraction', 'lifestyle', 'yourself', 'broken-heart', 'endurance', 'living-life', 'inspiring-quotes', 'charity', 'unrequited-love', 'liberation', 'moments', 'entrepreneur', 'attention', 'wise-words-quotes', 'gifts', 'terrorism', 'gandhi', 'rape', 'gospel', 'bad', 'zombies', 'fighting', 'time-travel', 'customer-service-quotes', 'love-quotations', 'insight', 'successful', 'parenthoodkissing', 'innocence', 'balance', 'self-development', 'cosmic-ordering', 'anarchy', 'self-determination', 'different', 'sales', 'enemy', 'morals', 'denial', 'self-growth', 'oneness', 'intimacy', 'fantasy-fiction', 'manifesting', 'challenge', 'song', 'enemies', 'free-will', 'writers-quotes', 'problem', 'libertarian', 'brave', 'best', 'college', 'chance', 'motivational-speaker-susan-young', 'rulesteen', 'time-management', 'wife', 'leadership-development', 'your-life', 'healthy-living', 'behavior', '물뽕', 'family-relationships', 'film', 'facts', 'injustice', 'الحب', 'envy', 'fighting-spirit', 'bdsm', 'celebration-quotes', 'mothers', 'oppression', 'achieve', 'development', 'authority', 'branding', '최음제', 'purpose-of-life', 'dignity', 'afterlife', '경마결과', 'create', 'synchronicityrunning', 'bliss', 'security', 'wise-sayings-quotes', 'شعر', 'aphorism', 'tolerance', 'insightful', 'reading-books', 'decision', 'worth', '专业办理海外文凭', 'movies', 'fact', 'inspirational-quotations', 'soulmates', 'boredom', 'breakup', 'caring', 'serving-god', 'lying', 'buddha', 'fall', 'conversation', 'confusion', 'mortality', 'miraboli', 'trees', 'motivation-quotes', 'witmotivation-quotes', 'praise', 'prison', 'wit', 'feminist', 'be-yourself', 'teenagers', 'forgive', 'escape', 'politicians', 'journalism', 'opinion', 'peace-of-mind', 'child-abuse', 'witty', 'sociology', 'voice', 'satan', 'leadership-quotes', 'proverbs', 'optimistic', 'boys', 'mountain', 'intellect', '사설경마배팅', 'angel', 'possibility', 'motivational-quote-of-the-day', 'end', 'visualizationlibraries', 'wish', 'success-strategies', '수면제가격', 'failure-quotes', 'country', 'hopeful', 'guru-quotes', 'change-your-life', 'legacy', 'possible', 'city', 'information', 'adversity-quotes', 'safety', 'novels', 'surrender', 'nothing', 'human-rights', 'diversity', 'atheist', 'perfect', 'social', 'today', 'hilarious', 'quote-about-life', 'activism', 'monsters', 'blame', 'rememberobsession', '바카라게임', 'remember', 'blame', 'understand', 'laugh', '수면제판매', 'overcomer', 'prose', 'play', 'daily-life', 'self-help-quotes', 'talents', 'quotes-for-instagram', 'united-states', 'quote-of-the-week', 'paris', 'cancer', 'self-respect', 'regrets', 'metaphysics', 'writing-craft', 'resilience', 'repentance', 'vanity', 'riches', 'self-knowledge', 'rights', 'christian-quotes', 'know']

'''------------------Code for generating Tag List------------------------
#Note: It takes over 5 min to get all tags..
def getlisttags():
    start = 1
    end = 34
    base_url = "https://www.goodreads.com/tag/popular/quotes?page="
    all_tags = []
    string_tag=""
    for i in range(34):
        complete_url = "https://www.goodreads.com/tag/popular/quotes?page="+str(i+1)
        print(complete_url)
        data = getResponse(complete_url)
        soup = BeautifulSoup(data, 'html.parser')
        tag_list = [tag.text for tag in soup.find_all('a', {'class': 'actionLinkLite'})]
        string_tag+=" | ".join(tag_list)
    #print(string_tag)
    all_tags = string_tag.split(" | ")
    print(all_tags)
    #return all_tags
-----------------------------End Code---------------------------------'''


#scrapes quotes from first page of goodreads.com
def getTopPopularQuotes(url):
    data = getResponse(url)
    soup = BeautifulSoup(data, 'html.parser')
    quotes_list = [quote.text.strip().split("\n")[0] for quote in soup.find_all('div', {'class': 'quoteText'})]
    author_list = [quote.text.strip().split("\n")[2] for quote in soup.find_all('div', {'class': 'quoteText'})]
    combine_list = []
    temp = ""
    for q, a in zip(quotes_list, author_list):
        temp = "" + q.strip(' ') + " -- " + a.strip(' ')
        combine_list.append(temp)
    for ele in combine_list:
        print(ele)
    writeToJSON(combine_list)

#scrapes recently added quotes from first page of goodreads.com
def getTopRecentlyAddedQuotes(url):
    complete_url = url +"/recently_added"
    data = getResponse(complete_url)
    soup = BeautifulSoup(data, 'html.parser')
    quotes_list = [quote.text.strip().split("\n")[0]
                   for quote in soup.find_all('div', {'class': 'quoteText'})]
    author_list = [quote.text.strip().split("\n")[2]
                   for quote in soup.find_all('div', {'class': 'quoteText'})]
    combine_list = []
    temp = ""
    for q, a in zip(quotes_list, author_list):
        temp = "" + q.strip(' ') + " -- " + a.strip(' ')
        combine_list.append(temp)
    for ele in combine_list:
        print(ele)
    writeToJSON(combine_list)

#scrapes quotes by user-defined tags..
def getTopQuotesByUserTag(url):
    print("To view complete list of tags press '' or visit goodreads.com")
    tag = str(input("Tag please.."))
    if tag in tags_list:
        print("Tag found.")
        print("Getting top quotes for the tag: "+tag)   
        complete_url = url +"/tag/"+tag
        data = getResponse(complete_url)
        soup = BeautifulSoup(data, 'html.parser')
        quotes_list = [quote.text.strip().split("\n")[0]
                       for quote in soup.find_all('div', {'class': 'quoteText'})]
        author_list = [quote.text.strip().split("\n")[2]
                       for quote in soup.find_all('div', {'class': 'quoteText'})]
        combine_list = []
        temp = ""
        for q, a in zip(quotes_list, author_list):
            temp = "" + q.strip(' ') + " -- " + a.strip(' ')
            combine_list.append(temp)
        for ele in combine_list:
            print(ele)
        writeToJSON(combine_list)    
    else:
        print("Tag not found.")

#scrapes quotes by page numbers..
def getQuotesByPageNumber(url):
    print("Enter specific page number to get quotes from the specific page..")
    page = int(input("Page number please.."))
    if page in range(100):
        print("Page found.")
        print("Getting top quotes for the page: "+str(page))   
        complete_url = url +"/?page="+str(page)
        data = getResponse(complete_url)
        soup = BeautifulSoup(data, 'html.parser')
        quotes_list = [quote.text.strip().split("\n")[0]
                       for quote in soup.find_all('div', {'class': 'quoteText'})]
        author_list = [quote.text.strip().split("\n")[2]
                       for quote in soup.find_all('div', {'class': 'quoteText'})]
        combine_list = []
        temp = ""
        for q, a in zip(quotes_list, author_list):
            temp = "" + q.strip(' ') + " -- " + a.strip(' ')
            combine_list.append(temp)
        for ele in combine_list:
            print(ele)
        writeToJSON(combine_list)
    else:
        print("Tag not found.")


#Not implemented!!
def getTopQuotesByAuthor(url):

    #Realised lately that its not possible..There is a random number(u_id) associated with each author.
    #we will have to create a map of each u_id associated with author and then use a list to retrive data from author.
    '''
    print("There are too many authors..")
    print("Visit 'https://www.goodreads.com/author/on_goodreads' to get list all author details..")
    print("Select author and proceed below..")
    author = str(input("Author please.."))
    '''
    '''
    complete_url = url +"/tag/"+
    data = getResponse(complete_url)
    soup = BeautifulSoup(data, 'html.parser')
    quotes_list = [quote.text.strip().split("\n")[0]
                   for quote in soup.find_all('div', {'class': 'quoteText'})]
    author_list = [quote.text.strip().split("\n")[2]
                   for quote in soup.find_all('div', {'class': 'quoteText'})]
    combine_list = []
    temp = ""
    for q, a in zip(quotes_list, author_list):
        temp = "" + q.strip(' ') + " -- " + a.strip(' ')
        combine_list.append(temp)
    for ele in combine_list:
        print(ele)
    '''



def main():
    base_url = "https://www.goodreads.com/quotes"
    
    print("Enter choice..")
    print("1. Top Popular Quotes..")
    print("2. Top Recent Quotes..")
    print("3. Top Quotes by Tag..")
    print("4. Top Quotes by Pagination..")

    choice = str(input(".."))
    
    if choice == "1":
        getTopPopularQuotes(base_url)
    elif choice == "2":    
        getTopRecentlyAddedQuotes(base_url)
    elif choice == "3":
        getTopQuotesByUserTag(base_url)
    elif choice == "4":    
        getQuotesByPageNumber(base_url)
    else:
        print("invalid choice..exiting..")
        time.sleep(1)
        exit()

    #getTopQuotesByAuthor()

if __name__ == '__main__':
    main()