32_stock_scraper.py

import urllib.request
from bs4 import BeautifulSoup


def get_stock_tickers():
    req = urllib.request.Request(
        "http://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
    )
    page = urllib.request.urlopen(req)
    soup = BeautifulSoup(page, "html.parser")
    table = soup.find("table", {"class": "wikitable sortable"})
    tickers = []
    for row in table.findAll("tr"):
        col = row.findAll("td")
        if len(col) > 0:
            tickers.append(str(col[0].string.strip()))
    tickers.sort()
    return tickers


def get_stock_prices(ticker_list):
    for ticker in ticker_list:
        htmlfile = urllib.request.urlopen(
            "http://finance.yahoo.com/q?s={0}".format(ticker)
        )
        htmltext = htmlfile.read()
        soup = BeautifulSoup(htmltext, "html.parser")
        htmlSelector = "yfs_l84_{0}".format(ticker.lower())
        for price in soup.find_all(id=htmlSelector):
            print("{0} is {1}".format(ticker, price.text))


def main():
    all_tickers = get_stock_tickers()
    get_stock_prices(all_tickers)


if __name__ == "__main__":
    main()