Permalink
Browse files

Exceptionable functions and refactorings

  • Loading branch information...
1 parent 96138e0 commit 7397e450e2e3eeed1177433a3c7e398bb860db11 @9seconds committed Dec 21, 2011
View
@@ -25,7 +25,7 @@
from utils.website import TwoStep, parser
-from utils.papercuts import convert_date, stripped, urlopen
+from utils.papercuts import convert_date, stripped, exceptionable, urlopen
@@ -34,17 +34,20 @@ class DISNews (TwoStep):
@staticmethod
@stripped
+ @exceptionable
def get_title (info):
return info.cssselect('h1.title')[0].text_content()
@staticmethod
@stripped
+ @exceptionable
def get_author (info):
return info.cssselect('.content .byline .b_author a.author')[0].text
@staticmethod
+ @exceptionable
def get_date (info):
return convert_date(
info.cssselect('.content .byline .date')[0].text
View
@@ -25,7 +25,7 @@
from utils.website import TwoStep, parser
-from utils.papercuts import convert_date, stripped, urlopen
+from utils.papercuts import convert_date, stripped, exceptionable, urlopen
@@ -34,42 +34,50 @@ class DISReviews (TwoStep):
@staticmethod
@stripped
+ @exceptionable
def get_artist (info):
return info.cssselect('.hreview .release_header .release_title h1 a')[0].text
@staticmethod
@stripped
+ @exceptionable
def get_album (info):
return info.cssselect('.hreview .release_header .release_title h1 a')[1].text
@staticmethod
@stripped
+ @exceptionable
def get_label (info):
return info.cssselect('.hreview .label_tokens .token a b')[0].text
@staticmethod
+ @exceptionable
def get_releasedate (info):
return convert_date(
- info.cssselect('.hreview .release_header .release_details')[0].text_content().strip().split()[-1]
+ info.cssselect('.hreview .release_header .release_details')[0].text_content()\
+ .strip().split()[-1]
)
@staticmethod
@stripped
+ @exceptionable
def get_author (info):
return info.cssselect('.hreview .b_author .author')[0].text
@staticmethod
+ @exceptionable
def get_pubdate (info):
return convert_date(
info.cssselect('.hreview .byline .date')[0].text
)
@staticmethod
+ @exceptionable
def get_score (info):
el = info.cssselect('.ratings .mark .value')
return int(el[0].text.strip()) \
@@ -101,6 +109,7 @@ def get_page_data (self, url, content):
return (url, artist, album, label, release_date, pub_date, author, score)
+
def get_pagecount (self):
handler = urlopen(self.page_counter.construct_url(1))
pagination = parser(handler.read()).cssselect('#content .pagination a')
View
@@ -25,24 +25,27 @@
from utils.website import OneStep
-from utils.papercuts import convert_date, stripped
+from utils.papercuts import convert_date, stripped, exceptionable
class FactNews (OneStep):
+ @exceptionable
def get_url (self, el):
return el.cssselect('a.archiveTitle')[0].get('href')
@staticmethod
@stripped
+ @exceptionable
def get_title (el):
return el.cssselect('h2')[0].text_content()
@staticmethod
+ @exceptionable
def get_date (el):
return convert_date(
el.cssselect('.postmetadata')[0].text_content().split('|')[0]
@@ -27,42 +27,47 @@
from decimal import Decimal
from utils.website import TwoStep
-from utils.papercuts import convert_date, stripped
+from utils.papercuts import convert_date, stripped, exceptionable
class FactReviews (TwoStep):
@staticmethod
+ @exceptionable
def get_aa_line (info):
- album = info.cssselect('.category-reviews h9 i')[0].text_content().strip()
+ album = info.cssselect('.category-reviews h9 i')[0].text_content().strip()
whole_line = info.cssselect('.category-reviews h9')[0].text_content()
- artist = whole_line.replace(album, '').strip()
+ artist = whole_line.replace(album, '').strip()
return (artist[:-1], album)
@staticmethod
@stripped
+ @exceptionable
def get_artist (info):
return FactReviews.get_aa_line(info)[0]
@staticmethod
@stripped
+ @exceptionable
def get_album (info):
return FactReviews.get_aa_line(info)[1]
@staticmethod
@stripped
+ @exceptionable
def get_label (info):
ret = info.findall('p')[1].text_content().replace('Available on:', '').split()
return ' '.join(ret[:-1])
@staticmethod
+ @exceptionable
def get_date (info):
return convert_date(
info.cssselect('#greyBox')[0].text_content().split(',')[1].split('and')[0]
@@ -71,11 +76,13 @@ def get_date (info):
@staticmethod
@stripped
+ @exceptionable
def get_author (info):
return info.findall('p')[-2].text_content()
@staticmethod
+ @exceptionable
def get_score (info):
have = lambda pattern: len(
info.cssselect( FactReviews.get_score_css(pattern) )
@@ -92,6 +99,7 @@ def get_score (info):
@staticmethod
+ @exceptionable
def get_score_css (number):
css = str(number).replace('.', '')
return 'table.record%s-article' % (css if not css.endswith('0') else css[:-1])
@@ -25,32 +25,36 @@
from utils.website import OneStep
-from utils.papercuts import convert_date, stripped
+from utils.papercuts import convert_date, stripped, exceptionable
class GorillaVsBearNews (OneStep):
+ @exceptionable
def get_url (self, el):
return el.cssselect('.postInfo h2 a')[0].get('href')
@staticmethod
@stripped
+ @exceptionable
def get_title (el):
return el.cssselect('.postInfo .pagetitle')[0].text_content()
@staticmethod
@stripped
+ @exceptionable
def get_author (el):
return el.cssselect(
'.entry .postmetadataBottom'
)[0].text_content().strip().split()[2]
@staticmethod
+ @exceptionable
def get_date (el):
return convert_date(
el.cssselect('.postInfo .pageTitleR')[0].text_content()
View
@@ -25,19 +25,21 @@
from utils.website import OneStep
-from utils.papercuts import convert_date, stripped
+from utils.papercuts import convert_date, stripped, exceptionable
class P4News (OneStep):
@staticmethod
+ @exceptionable
def get_titlelink (el):
return el.cssselect('h3.title a')[0]
@staticmethod
+ @exceptionable
def get_posted (el):
content = el.cssselect('.posted-at')[0].text_content().strip().split()
return (
@@ -48,17 +50,20 @@ def get_posted (el):
@staticmethod
@stripped
+ @exceptionable
def get_title (el):
return P4News.get_titlelink(el).text_content()
@staticmethod
@stripped
+ @exceptionable
def get_author (el):
return P4News.get_posted(el)[0]
@staticmethod
+ @exceptionable
def get_date (el):
return convert_date(P4News.get_posted(el)[1])
View
@@ -24,8 +24,10 @@
+from decimal import Decimal
+
from utils.website import TwoStep
-from utils.papercuts import convert_date, stripped
+from utils.papercuts import convert_date, stripped, exceptionable
@@ -34,6 +36,7 @@ class P4Reviews (TwoStep):
@staticmethod
@stripped
+ @exceptionable
def get_artist (info):
return info.find('h1').find('a').text \
if len(info.find('h1')) > 0 \
@@ -42,37 +45,43 @@ def get_artist (info):
@staticmethod
@stripped
+ @exceptionable
def get_album (info):
return info.find('h2').text
@staticmethod
@stripped
+ @exceptionable
def get_label (info):
return info.find('h3').text.split(';')[0]
@staticmethod
@stripped
+ @exceptionable
def get_year (info):
year = info.find('h3').text.split(';')[1]
return year.split('/')[1] if '/' in year else year
@staticmethod
@stripped
+ @exceptionable
def get_author (info):
return info.find('div')[0].text.replace('By', '').split(';')[0]
@staticmethod
+ @exceptionable
def get_date (info):
return convert_date(info.find('div')[0].text.split(';')[1])
@staticmethod
+ @exceptionable
def get_score (info):
- return float(info.findall('div')[1].findall('span')[0].text.strip())
+ return Decimal(info.findall('div')[1].findall('span')[0].text.strip())
def __init__ (self, output = None):
Oops, something went wrong.

0 comments on commit 7397e45

Please sign in to comment.