Skip to content
Browse files

Launcher is ready, autoregister plugin system draft

  • Loading branch information...
1 parent 0e06dc4 commit d3816e3c148e2b35e78a40dbc4aac21d13719723 @9seconds committed Dec 17, 2011
Showing with 123 additions and 8 deletions.
  1. +32 −5 main.py
  2. +62 −0 plugins/__init__.py
  3. +4 −0 plugins/disnews.py
  4. +4 −0 plugins/disreviews.py
  5. +4 −0 plugins/gorillavsbearnews.py
  6. +4 −0 plugins/p4news.py
  7. +4 −0 plugins/p4reviews.py
  8. +9 −3 website.py
View
37 main.py
@@ -22,11 +22,38 @@
#
#
-# check lxml and dateutil
+
+
+import sys
+
+def err (module_name, package_name):
+ sys.stderr.write(
+ "{0} module is unavaliable. Please install it\n(e.g. 'sudo apt-get install {1}')\n".format(
+ module_name,
+ package_name
+ ))
+ sys.exit(2)
+
+try:
+ import lxml
+except ImportError:
+ err('lxml', 'python-lxml')
+
+try:
+ import dateutil
+except ImportError:
+ err('dateutil', 'python-dateutil')
+
+
+
from plugins import *
+import plugins
+
+
-h = [disnews.DISNews, p4reviews.P4Reviews, p4news.P4News, disreviews.DISReviews, gorillavsbearnews.GorillaVsBearNews]
-h = [p4news.P4News]
-for p in h:
- p().handle()
+try:
+ for site in plugins.get_plugins(sys.argv[1:]):
+ site().handle()
+except ValueError as e:
+ print e
View
62 plugins/__init__.py
@@ -22,10 +22,72 @@
#
#
+
+
+import random
+
+
+
__all__ = (
'p4reviews',
'disreviews',
'p4news',
'disnews',
'gorillavsbearnews'
)
+
+_plugins = {} # {plugin_name : plugin_class}
+
+
+
+def register (plugin_class, *plugin_names):
+ for name in plugin_names:
+ if name in _plugins:
+ raise KeyError(name)
+
+ for name in plugin_names:
+ _plugins[name] = plugin_class
+
+
+def unregister(*plugin_names):
+ for name in plugin_names:
+ try:
+ del _plugin[name]
+ except KeyError:
+ pass
+
+
+def have_plugin (plugin_name):
+ return plugin_name in _plugins
+
+
+def get_unknown_plugins (plugin_names):
+ return filter(
+ lambda name: not have_plugin(name),
+ plugin_names
+ )
+
+
+def going_for_all (plugin_names):
+ return len(plugin_names) == 1 and 'all' == plugin_names[0]
+
+
+def prepared (plugs):
+ ret = list(frozenset(plugs))
+ random.shuffle(ret) # trick to decrease load on particular service
+ return ret
+
+
+def get_registered_plugins (plugin_names):
+ return ( _plugins[name] for name in plugin_names )
+
+
+def get_plugins (plugin_names):
+ if going_for_all (plugin_names):
+ return prepared(_plugins.values())
+ else:
+ unknown = get_unknown_plugins(plugin_names)
+ if len(unknown) == 0:
+ return prepared(get_registered_plugins(plugin_names))
+ else:
+ raise ValueError('Unknown plugins: {0}'.format(str(unknown)))
View
4 plugins/disnews.py
@@ -77,3 +77,7 @@ def get_pagecount (self):
pagination = website.parser(handler.read()).cssselect('#content .pagination a')
handler.close()
return int(pagination[-2].text)
+
+
+
+DISNews.register()
View
4 plugins/disreviews.py
@@ -110,3 +110,7 @@ def get_pagecount (self):
def get_sorter (self, tupl):
return tupl[1] + tupl[2]
+
+
+
+DISReviews.register()
View
4 plugins/gorillavsbearnews.py
@@ -79,3 +79,7 @@ def handle_element(self, element):
date = self.get_date(element)
return (url, title, author, date)
+
+
+
+GorillaVsBearNews.register()
View
4 plugins/p4news.py
@@ -89,3 +89,7 @@ def handle_element(self, element):
date = self.get_date(element)
return (url, title, author, date)
+
+
+
+P4News.register()
View
4 plugins/p4reviews.py
@@ -102,3 +102,7 @@ def get_page_data (self, url, content):
def get_sorter (self, tupl):
return tupl[1] + tupl[2]
+
+
+
+P4Reviews.register()
View
12 website.py
@@ -35,9 +35,10 @@
import workerconsumerpool as wcp
import pagecounter
import utils
+import plugins
from lxml.html import document_fromstring as parser
-from lxml.html import tostring as parser_str
+
def stripped (func):
@@ -75,6 +76,11 @@ def get_page_content (url):
return content
+ @classmethod
+ def register (cls):
+ plugins.register(cls, cls.__name__.lower())
+
+
def __init__ (
self,
domain,
@@ -107,8 +113,8 @@ def handle (self):
print 'Handling {0}'.format(self.task_name)
print '{0} pages to handle'.format(self.get_pagecount())
- #for page in xrange(self.page_counter.left_bound, self.get_pagecount()+1):
- for page in xrange(self.page_counter.left_bound, 5):
+ for page in xrange(self.page_counter.left_bound, self.get_pagecount()+1):
+ #for page in xrange(self.page_counter.left_bound, 5):
parse_results = None
for attempt in xrange(self.tries):
parse_results = self._parse_linkpage(page)

0 comments on commit d3816e3

Please sign in to comment.
Something went wrong with that request. Please try again.