Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 119 lines (103 sloc) 3.89 KB
#!/usr/bin/env python3
import sys
import os
from time import sleep
from requests import get
from bs4 import BeautifulSoup
from dateutil.parser import parse
import datetime
import sqlite3
# sqlite globals
def dbinit(dbname):
global conn
conn = sqlite3.connect(dbname)
global c
c = conn.cursor()
# open the db if it exists, create and initialize if not
def dbopen(dbname):
if os.path.exists(dbname):
dbinit(dbname)
else:
dbinit(dbname)
createtables(dbname)
# initializing the place where the wigs live
def createtables(dbname):
c.execute('drop table if exists wiggames')
c.execute('''CREATE TABLE wiggames
(gameid integer, gamedate integer, gametype text, gamelength integer, gamemap text)''')
conn.commit()
# returns either the entire row of a gameid, or None if the gameid doesn't exist in the database
def checkforgameid(gameid):
c.execute('SELECT * FROM wiggames WHERE gameid={thisid}'.format(thisid=gameid))
return c.fetchone()
# check if a gameid already exists in the database. if not, query battle.net for its game page and return the html
def downloadwig(gateway, gameid):
if gateway == 'Kalimdor':
baseurl = 'http://asialadders.battle.net/war3/ladder/w3xp-game-detail.aspx'
else:
baseurl = 'http://classic.battle.net/war3/ladder/w3xp-game-detail.aspx'
fullurl = baseurl + '?Gateway=' + gateway + '&GameID=' + str(gameid)
if checkforgameid(gameid):
return {'status': 'exists', 'html': None}
try:
response = get(fullurl)
except:
sleep(10)
response = get(fullurl)
if 'errormessage.html' in response.url:
return {'status': 'error', 'html': None}
else:
return {'status': 'saved', 'html': response.text}
# take a game page html blob and slice out the metadata that we care about. adapted from wigfo.py
def parsegame(gameid, html):
container = BeautifulSoup(html, "html.parser").find("table", class_="mainTable")
basicinfo = container.find_all("td", class_="playerStatsDataLeft")
gamedate = int(parse(''.join(basicinfo[0].strings)).strftime('%s')) # convert the date to unix epoch
gamemap = basicinfo[1].string
gametype = basicinfo[2].string
gamelength = int(basicinfo[3].string.split(" ")[0])
return [gameid, gamedate, gametype, gamelength, gamemap]
def writegametodb(gamerow):
c.execute('''INSERT INTO wiggames VALUES (?,?,?,?,?)''', gamerow)
conn.commit()
def main():
try:
gateway = sys.argv[1]
assert gateway in ['Azeroth', 'Northrend', 'Lordaeron', 'Kalimdor']
dbname = gateway + '.db'
except:
print('usage: %s [gateway] [starting gameid]' % sys.argv[0])
sys.exit(1)
dbopen(dbname)
try: # start from user-specified gameid
startid = int(sys.argv[2])
except: # or start from the highest gameid in the db
c.execute('select gameid from wiggames order by gameid desc limit 1')
startid = c.fetchone()[0]
gameid = startid
consecutiveerrorlimit = 5
errors = 0
while errors < consecutiveerrorlimit:
print(gameid, gateway, 'scraping...', end='')
scrape = downloadwig(gateway, gameid)
if scrape['status'] == 'error':
print(scrape['status'])
errors += 1
elif scrape['status'] == 'saved':
try:
print('saved ', end='')
gamerow = parsegame(gameid, scrape['html'])
date = datetime.datetime.fromtimestamp(gamerow[1], datetime.timezone.utc)
writegametodb(gamerow)
print(date, gamerow[2])
errors = 0 # reset error count after successful scrape
except:
gameid -= 1
errors += 1
sleep(10)
elif scrape['status'] == 'exists':
print(scrape['status'])
gameid += 1
conn.close()
if __name__ == '__main__':
main()