Skip to content
Find file
Fetching contributors…
Cannot retrieve contributors at this time
83 lines (68 sloc) 2.37 KB
from common import *
from re import compile, DOTALL, MULTILINE
from urlgrab import Cache
from urlparse import urljoin
linkPattern = compile("<h3><a href=\"(/[^\"]+)\">(.+?)</a></h3>")
earlierPattern = compile("<a href='([^\']+)'>.+?Earlier Stories.+?</a>", DOTALL | MULTILINE)
titlePattern = compile("<h2>(.+?)</h2>")
subtitlePattern = compile("<p class=\"standfirst\">(.+?)</p>")
contentPattern = compile("<strong class=\"trailer\">.+?</p>(.+?)(?:(?:<p>(?:(?:<i>)|(?:<small>)|(?:<font size=\"-2\">)|(?:<br>\n))?BOFH .+? Simon Travaglia)|(?:<ul class=\"noindent\">)|(?:<ul>.+?<li><a href=\"http://www.theregister.co.uk/content/30/index.html\">BOFH: The whole shebang</a></li>)|(?:</form>))", DOTALL| MULTILINE)
adPattern = compile("(<div id=ad-mu1-spot>.+?</div>)", MULTILINE | DOTALL)
episodePattern = compile("<strong class=\"trailer\">Episode \d+")
url = "http://www.theregister.co.uk/data_centre/bofh/"
pages = [url]
cache = Cache()
while True:
print url
data = cache.get(url).read()
links = linkPattern.findall(data)
if links == []:
break
pages.insert(0, url)
earlier = earlierPattern.findall(data)
url = urljoin(url, earlier[0])
skipTitles = ["Salmon Days is Go!"]
year = None
newItems = False
for mainPage in pages:
data = cache.get(mainPage).read()
links = linkPattern.findall(data)
links.reverse()
for l in links:
url = urljoin(mainPage, l[0])
newyear = url.split("/")[3]
if newyear != year:
if year != None:
if int(newyear) < int(year):
raise Exception, (year, newyear)
tocEnd(toc)
makeMobi(folder, "Simon Travaglia", newitems = newItems)
newItems = False
folder = "BOFH-%s"%newyear
toc = tocStart(folder)
year = newyear
data = cache.get(url, max_age = -1).read()
episode = episodePattern.findall(data)
if len(episode) == 0:
print "Skipping", url
continue
print url
title = titlePattern.findall(data)[0]
print title
if title in skipTitles:
print "skipping", title
continue
subtitle = subtitlePattern.findall(data)[0]
content = contentPattern.findall(data)[0]
ad = adPattern.findall(data)[0]
content = content.replace(ad, "")
content = content.decode('utf-8')
title = title.decode("utf-8")
subtitle = subtitle.decode("utf-8")
assert len(content)>0
if generatePage(url, title, subtitle + "<br />\n" + content, folder, toc):
newItems = True
#break
print links
tocEnd(toc)
makeMobi(folder, "Simon Travaglia")
Something went wrong with that request. Please try again.