Permalink
Browse files

you can now skip the parser step by using SkipParser

  • Loading branch information...
1 parent 740d7da commit 41d918244a8bdf3f74159e4ca03bae1e51a3414d @erowan erowan committed Jan 31, 2012
Showing with 29 additions and 5 deletions.
  1. +22 −0 .gitignore
  2. +7 −5 src/pull/site.py
View
@@ -0,0 +1,22 @@
+*.#*
+*.pyc
+*.pt.py
+*.cache
+.coverage
+src/*/*.egg-info/*
+*.log*
+*.DS_Store
+.Python
+.installed.cfg
+.project
+.pydevproject
+bin/*
+develop-eggs/*
+eggs/*
+include/*
+lib*/*
+parts/*
+docs/_build/*
+*.cov2emacs.log
+*~
+*.swp
View
@@ -124,13 +124,13 @@ def fetch(self, files):
'''
cache_files = []
failures = []
- for url, file in files:
+ for url, f in files:
try:
log.info("Downloading: " + str(url))
response, headers = self.fetch_url(url)
log.debug('response headers=%s' % str(headers))
- write_cache_file(response, file)
- cache_files.append(file)
+ write_cache_file(response, f)
+ cache_files.append(f)
except Exception, e:
# todo: think about passing failures back by setting
# site.stats[feed]['errors'] because if len(cache_files) > 0
@@ -219,11 +219,12 @@ def parse(self, file_path):
Parse the contents of file into a sequence of dicts.
@param file_path: path to file to parse.
'''
- pass
+ return file_path
def get_logger(self):
return logging.getLogger(self.name)
+SkipParser = Parser
class Feed(object):
'''
@@ -334,7 +335,8 @@ def __parse(self, cache_files):
self.get_logger().info('Parsing file %s' % file_path)
yield parser.parse(file_path)
-def build_feed(name, protocol, parser, **kwargs):
+def build_feed(name, protocol, parser=None, **kwargs):
+ parser = parser or SkipParser()
return Feed(name, protocol, parser, **kwargs)
class Site(object):

0 comments on commit 41d9182

Please sign in to comment.