-
Notifications
You must be signed in to change notification settings - Fork 0
/
admin.py
46 lines (42 loc) · 1.84 KB
/
admin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
from common import SOURCE_URL, HOSTNAME, BASE_PATH
from datetime import datetime
from google.appengine.ext import db
from models import Item, ChangeDate, CHANGE_DATE_KEY
from util import cachingFetch, cachingFetches
from xml.sax.saxutils import escape
import feeds
import itertools
import re
LINE_PATTERN = r'\s*(\d\d\d):\s*<a href="(.*mp3)".*?>(.*)</a>(.*)<br>'
def updateStore():
page = cachingFetch(SOURCE_URL)
lastmodified = page.getheader('last-modified')
if lastmodified != None:
date = datetime.strptime(lastmodified, "%a, %d %b %Y %H:%M:%S %Z")
else:
date = datetime.now()
ChangeDate(key_name=CHANGE_DATE_KEY, date=date).put()
tuples = (m.groups() for m in (re.match(LINE_PATTERN, line) for line in
page.readlines()) if m != None)
urls = dict(("http://" + HOSTNAME + BASE_PATH + tup[1], tup)
for tup in tuples)
responses = cachingFetches(urls.iterkeys(),
methods=itertools.repeat("HEAD"),
trustCache=True)
db.put([makeItem(urls[url], responses[url]) for url in urls.iterkeys()])
feeds.clearCache()
def makeItem(tup, response):
mp3path = BASE_PATH + tup[1]
mp3url = "http://" + HOSTNAME + mp3path
guid = makeGuid(mp3path)
return Item(key_name=guid,
title=tup[0],
link=mp3url,
description=escape(tup[2].rstrip() + tup[3].rstrip()),
pubDate=datetime.strptime(response.getheader('last-modified'),
"%a, %d %b %Y %H:%M:%S %Z"),
length=int(response.getheader('content-length')),
mimeType=response.getheader('content-type'),
guid=guid)
def makeGuid(mp3path):
return "tag:lukeyear.iheardata.com," + "2010-03-12" + ":" + HOSTNAME + mp3path