-
Notifications
You must be signed in to change notification settings - Fork 0
/
Feed.py
33 lines (26 loc) · 792 Bytes
/
Feed.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import datetime
import Page
from TrademeItem import *
from google.appengine.ext import db
class Feed(db.Model):
created_at = db.DateTimeProperty(auto_now_add=True)
url = db.StringProperty()
title = db.StringProperty()
def load(self):
self.start_time = datetime.now()
self.max_time = timedelta(seconds=2)
self.page = Page.fetch(self.url, 5)
if(self.page):
soup = BeautifulSoup(self.page)
items = soup.findAll('li', {"class": re.compile('.*listingCard.*')})
self.items = map(self.process_item, items)
for item in self.items:
item.load_details()
time = datetime.now()
if(time > self.start_time + self.max_time):
# Stop due to time contraint
break
return self.items
def process_item(self, item):
titem = TrademeItem(item)
return titem