Browse files

call biblio in alias thread; closes #64

  • Loading branch information...
1 parent 902973a commit bbcff8fdd00259be3db5bd673998c6e9936adfa5 @hpiwowar hpiwowar committed Apr 16, 2012
View
17 test/integration_tests/test_alias_queue.py
@@ -12,6 +12,8 @@
datadir = os.path.join(os.path.split(__file__)[0], "../data")
DRYAD_CONFIG_FILENAME = "totalimpact/providers/dryad.conf.json"
+SAMPLE_EXTRACT_BIBLIO_PAGE = os.path.join(datadir,
+ "sample_extract_biblio_page.xml")
SAMPLE_EXTRACT_ALIASES_PAGE = os.path.join(datadir,
"sample_extract_aliases_page.xml")
@@ -22,9 +24,13 @@ def __init__(self, status, content):
self.text = content
def get_aliases_html_success(self, url, headers=None, timeout=None):
- f = open(SAMPLE_EXTRACT_ALIASES_PAGE, "r")
+ if ("dc.contributor" in url):
+ f = open(SAMPLE_EXTRACT_BIBLIO_PAGE, "r")
+ else:
+ f = open(SAMPLE_EXTRACT_ALIASES_PAGE, "r")
return DummyResponse(200, f.read())
+
class TestAliasQueue(unittest.TestCase):
def setUp(self):
@@ -65,14 +71,16 @@ def test_alias_queue(self):
resp_dict = json.loads(response.data)
assert_equals(
set(resp_dict.keys()),
- set([u'created', u'last_requested', u'metrics', u'last_modified', u'biblio', u'id', u'aliases'])
+ set([u'created', u'last_requested', u'metrics', u'last_modified',
+ u'biblio', u'id', u'aliases'])
)
assert_equals(unicode(TEST_DRYAD_DOI), resp_dict["aliases"]["doi"])
# test the view works
res = self.d.view("aliases")
assert len(res["rows"]) == 1, res
- assert_equals(res["rows"][0]["value"]["aliases"]["doi"], TEST_DRYAD_DOI)
+ assert_equals(res["rows"][0]["value"]["aliases"]["doi"],
+ TEST_DRYAD_DOI)
# see if the item is on the queue
my_alias_queue = AliasQueue(self.d)
@@ -94,3 +102,6 @@ def test_alias_queue(self):
resp_dict["aliases"]["title"][0],
"data from: can clone size serve as a proxy for clone age? an exploration using microsatellite divergence in populus tremuloides"
)
+ assert_equals(resp_dict["biblio"]["year"], "2010")
+
+
View
4 test/unit_tests/provider_dryad/test_dryad.py
@@ -258,8 +258,8 @@ def test_09a_basic_extract_biblio(self):
def test_09b_get_biblio_success(self):
Provider.http_get = get_biblio_html_success
- biblio_object = self.provider.biblio(self.simple_item)
- assert_equals(biblio_object.data, {'year': u'2010', 'title': u'Data from: Can clone size serve as a proxy for clone age? An exploration using microsatellite divergence in Populus tremuloides'})
+ item = self.provider.biblio(self.simple_item)
+ assert_equals(item.biblio.data, {'year': u'2010', 'title': u'Data from: Can clone size serve as a proxy for clone age? An exploration using microsatellite divergence in Populus tremuloides'})
@raises(ProviderClientError)
def test_09c_biblio_400(self):
View
9 totalimpact/backend.py
@@ -179,13 +179,13 @@ def run(self, run_only_once=False):
def process_item_for_provider(self, item, provider, method):
""" Run the given method for the given provider on the given item
- method should either be 'aliases' or 'metrics'
+ method should either be 'aliases', 'biblio', or 'metrics'
This will deal with retries and sleep / backoff as per the
configuration for the given provider. We will return true if
the given method passes, or if it's not implemented.
"""
- if method not in ('aliases','metrics'):
+ if method not in ('aliases', 'biblio', 'metrics'):
raise NotImplementedError("Unknown method %s for provider class" % method)
log.info("Item %s: processing %s for provider %s" % (item, method, provider))
@@ -295,6 +295,11 @@ def process_item(self, item):
# total number of retries. Don't process any
# more providers, we abort this item entirely
break
+ if not self.process_item_for_provider(item, provider, 'biblio'):
+ # This provider has failed and exceeded the
+ # total number of retries. Don't process any
+ # more providers, we abort this item entirely
+ break
View
4 totalimpact/models.py
@@ -171,6 +171,10 @@ def __str__(self):
def __repr__(self):
return str(self.data)
+ def as_dict(self):
+ # renamed for consistancy with Items(); TODO cleanup old one
+ return self.data
+
class Metrics(object):
"""
View
3 totalimpact/providers/dryad.py
@@ -233,7 +233,8 @@ def provides_biblio(self):
def biblio(self, item):
id = self._get_dryad_doi(item)
biblio_object = self.get_biblio_for_id(id)
- return biblio_object
+ item.biblio = biblio_object
+ return item
def get_biblio_for_id(self, id):
url = self.config.biblio['url'] % id

0 comments on commit bbcff8f

Please sign in to comment.