forked from glebpopoff/Hacker-News-Droid-API
-
Notifications
You must be signed in to change notification settings - Fork 0
/
APIContent.py
148 lines (137 loc) · 6.08 KB
/
APIContent.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
import APIUtils
import DataCache
from UserString import MutableString
import logging
import AppConfig
#get cached content
def getCache(pageId,format):
logging.debug('getCache: %s' % pageId)
try:
dbData = DataCache.getData(pageId,format)
if (dbData):
if (DataCache.hasExpired(dbData)):
#data has expired, remove it
try:
dbData[0].delete()
return None
except:
logging.error('getCache: unable to remove cache')
return None
else:
logging.debug('getCache: got cached data for id %s' % id)
return dbData[0].rec_xml
except:
logging.error('getCache: unable to get/retrieve cache')
return None
#parse HN's submissions by user
def getHackerNewsSubmittedContent(user, format='json', url='', referer='', remote_addr=''):
#only cache homepage data
apiURL = "%s/submitted?id=%s" % (AppConfig.hackerNewsURL, user)
apiURLBackup = "%s/submitted?id=%s" % (AppConfig.hackerNewsURLBackup, user)
id = '/submitted/%s' % (user)
cachedData = None
cachedData = getCache(id,format)
if (cachedData):
return cachedData
else:
hnData = APIUtils.parsePageContent(apiURL,apiURLBackup, '/submitted', None,format)
if (hnData):
logging.debug('getHackerNewsSubmittedContent: storing cached value for id %s' % id)
DataCache.putData(id, format,APIUtils.removeNonAscii(hnData), url, referer, remote_addr)
return hnData
else:
logging.warning('getHackerNewsSubmittedContent: unable to retrieve data for id %s' % id)
return ''
#parse HN's comments by story id
def getHackerNewsComments(articleId, format='json', url='', referer='', remote_addr=''):
#only cache homepage data
apiURL = "%s/item?id=%s" % (AppConfig.hackerNewsURL, articleId)
apiURLBackup = "%s/item?id=%s" % (AppConfig.hackerNewsURLBackup, articleId)
id = '/comments/%s' % (articleId)
cachedData = getCache(id,format)
if (cachedData):
return cachedData
else:
hnData = APIUtils.parseCommentsContent(apiURL, apiURLBackup, '/comments', None,format)
if (hnData):
logging.debug('getHackerNewsComments: storing cached value for id %s' % id)
DataCache.putData(id, format,APIUtils.removeNonAscii(hnData), url, referer, remote_addr)
return hnData
else:
logging.warning('getHackerNewsComments: unable to retrieve data for id %s' % id)
return ''
#parse HN's comments by story id
def getHackerNewsNestedComments(articleId, format='json', url='', referer='', remote_addr=''):
#only cache homepage data
apiURL = "%s/item?id=%s" % (AppConfig.hackerNewsURL, articleId)
apiURLBackup = "%s/item?id=%s" % (AppConfig.hackerNewsURLBackup, articleId)
id = '/nestedcomments/%s' % (articleId)
#cache data
cachedData = getCache(id,format)
if (cachedData):
return cachedData
else:
try:
hnData = APIUtils.parseNestedCommentsContent(apiURL, apiURLBackup, '/nestedcomments', None,format)
if (hnData):
logging.debug('getHackerNewsComments: storing cached value for id %s' % id)
DataCache.putData(id, format,APIUtils.removeNonAscii(hnData), url, referer, remote_addr)
return hnData
else:
logging.warning('getHackerNewsComments: unable to retrieve data for id %s' % id)
return ''
except:
logging.warning('getHackerNewsComments: error(s) getting comments %s' % id)
return ''
def getHackerNewsSimpleContent(fetcherURL, fetcherBackupURL, id, page='', format='json', url='', referer='', remote_addr=''):
#don't cache paginated content
if (page):
return APIUtils.parsePageContent(fetcherURL, fetcherBackupURL, id, page,format)
else:
cachedData = getCache(id,format)
if (cachedData):
return cachedData
else:
hnData = APIUtils.parsePageContent(fetcherURL, fetcherBackupURL, id, page,format)
if (hnData):
logging.debug('getHackerNewsSimpleContent: storing cached value for id %s' % id)
DataCache.putData(id, format,APIUtils.removeNonAscii(hnData), url, referer, remote_addr)
return hnData
else:
logging.warning('getHackerNewsSimpleContent: unable to retrieve data for id %s' % id)
return ''
#parse HN's ask content
def getHackerNewsAskContent(page='', format='json', url='', referer='', remote_addr=''):
return getHackerNewsSimpleContent(AppConfig.hackerNewsAskURL, AppConfig.hackerNewsAskURLBackup, '/ask', page, format, url, referer, remote_addr)
#parse HN's best content
def getHackerNewsBestContent(page='', format='json', url='', referer='', remote_addr=''):
return getHackerNewsSimpleContent(AppConfig.hackerNewsBestURL, AppConfig.hackerNewsBestURLBackup, '/best', page, format, url, referer, remote_addr)
#parse HN's newest content
def getHackerNewsNewestContent(page='', format='json', url='', referer='', remote_addr=''):
return getHackerNewsSimpleContent(AppConfig.hackerNewsNewestURL, AppConfig.hackerNewsNewestURLBackup, '/newest', page, format, url, referer, remote_addr)
#get homepage second page stories
def getHackerNewsSecondPageContent(page='', format='json', url='', referer='', remote_addr=''):
return getHackerNewsSimpleContent(AppConfig.hackerNewsPage2URL, AppConfig.hackerNewsPage2URLBackup, '/news2', page, format, url, referer, remote_addr)
#get homepage first page stories
def getHackerNewsPageContent(page='', format='json', url='', referer='', remote_addr=''):
return getHackerNewsSimpleContent(AppConfig.hackerNewsURL, AppConfig.hackerNewsURLBackup, '/news', page, format, url, referer, remote_addr)
#get latest homepage stories
def getHackerNewsLatestContent(page='', format='json', url='', referer='', remote_addr='', limit=1):
#only cache homepage data
limit = int(limit)
if (page):
return APIUtils.parsePageContent(AppConfig.hackerNewsURL, AppConfig.hackerNewsURLBackup, '/latest', page,format,limit)
else:
id = '/latest/%s' % limit
cachedData = getCache(id,format)
if (cachedData):
return cachedData
else:
hnData = APIUtils.parsePageContent(AppConfig.hackerNewsURL,AppConfig.hackerNewsURLBackup, '/latest', page,format,limit)
if (hnData):
logging.debug('getHackerNewsLatestContent: storing cached value for id %s' % id)
DataCache.putData(id, format,APIUtils.removeNonAscii(hnData), url, referer, remote_addr)
return hnData
else:
logging.warning('getHackerNewsLatestContent: unable to retrieve data for id %s' % id)
return ''