Permalink
Browse files

Adicionada a capacidade na função ``datacollector.identify_changes`` …

…de identificar alterações em fascículos pertencentes a periódicos que fazem parte do catálogo
  • Loading branch information...
1 parent f0cb9d7 commit 26e04904043f510b087a2b52f41bffe7da3ee4bd @gustavofonseca committed Feb 28, 2013
Showing with 264 additions and 59 deletions.
  1. +2 −26 opac/utils/sync/datacollector.py
  2. +53 −0 opac/utils/tasks.py
  3. +0 −33 opac/utils/tests/tests_datacollector.py
  4. +209 −0 opac/utils/tests/tests_tasks.py
@@ -134,29 +134,5 @@ def get_all_collections(self):
"""
return self.iter_docs('collections')
-
-def identify_changes(changes, collections, journals):
- """
- Returns a list of ``object_uri`` that must be
- synced.
-
- ``changes`` is an iterable where each element is an
- entry in changes API.
-
- ``collections`` is an iterable of collections
- that must have all its journals synced.
-
- ``journals`` is an iterable of journals that must
- be synced.
- """
- colls_resource_uri = set(c.resource_uri for c in collections)
- jours_resource_uri = set(j.resource_uri for j in journals)
-
- changed = []
- for change_rec in changes:
- if change_rec.get('collection_uri') in colls_resource_uri or (
- change_rec.get('object_uri') in jours_resource_uri):
-
- changed.append(change_rec.get('object_uri'))
-
- return changed
+ def get_changes(self, since=0):
+ return self.iter_docs('changes')
View
@@ -7,6 +7,7 @@
from .sync import dataloader
from .sync import pipes
from catalog import models
+from catalog import mongomodels
def _get_user_catalog_definitions():
@@ -58,8 +59,13 @@ def _what_have_changed(managerapi_dep=datacollector.SciELOManagerAPI):
Returns an iterator containing all journals that must be created
or updated in order to keep the catalog updated.
"""
+ scielo_api = managerapi_dep(settings=settings)
full_collections, journals_a_la_carte = _get_user_catalog_definitions()
+ data = scielo_api.get_changes()
+
+ return identify_changes(data, full_collections, journals_a_la_carte)
+
@task(name='utils.tasks.build_catalog')
def build_catalog():
@@ -123,3 +129,50 @@ def sync_journals_meta(managerapi_dep=datacollector.SciELOManagerAPI):
name=col.get('title', ''),
collection=cols_memo[collection],
)
+
+
+def _list_issues_uri(journal_meta, journal_dep=mongomodels.Journal):
+ # TODO: This instantiation logic must be at Journal.get_journal
+ journal_data = journal_dep.objects.find_one({'id': journal_meta.resource_id})
+ journal_doc = journal_dep(**journal_data)
+ return (issue.resource_uri for issue in journal_doc.list_issues())
+
+
+def identify_changes(changes,
+ collections,
+ journals,
+ list_issues_uri_dep=_list_issues_uri):
+ """
+ Returns a list of ``object_uri`` that must be
+ synced.
+
+ ``changes`` is an iterable where each element is an
+ entry in changes API.
+
+ ``collections`` is an iterable of collections
+ that must have all its journals synced.
+
+ ``journals`` is an iterable of journals that must
+ be synced.
+ """
+ journals_list = []
+ issues_list = []
+
+ # list uris from all journals and its issues
+ for j in journals:
+ journals_list.append(j.resource_uri)
+ issues_list.append(list_issues_uri_dep(j))
+
+ collections_uris = (c.resource_uri for c in collections)
+ journals_uris = journals_list
+ issues_uris = itertools.chain(*issues_list)
+ super_set = set().union(collections_uris, journals_uris, issues_uris)
+
+ changed = set()
+ for change_rec in changes:
+ if change_rec.get('collection_uri') in super_set or (
+ change_rec.get('object_uri') in super_set):
+
+ changed.add(change_rec.get('object_uri'))
+
+ return list(changed)
@@ -83,36 +83,3 @@ def test_single_document_of_an_endpoint(self):
res = api.fetch_data('journals', resource_id=1)
self.assertIn('title', res)
-
-
-class ChangesIdentificationTests(mocker.MockerTestCase, TestCase):
-
- def test_identify_journals_given_collections(self):
- from utils.sync.datacollector import identify_changes
-
- changes = [
- {
- "changed_at": "2013-01-23T15:11:33.409478",
- "collection_uri": "/api/v1/collections/1/",
- "event_type": "updated",
- "object_uri": "/api/v1/journals/31/",
- "resource_uri": "/api/v1/changes/8/",
- "seq": 8
- },
- {
- "changed_at": "2013-01-23T15:12:33.409478",
- "collection_uri": "/api/v1/collections/2/",
- "event_type": "added",
- "object_uri": "/api/v1/issues/2840/",
- "resource_uri": "/api/v1/changes/2/",
- "seq": 9
- },
- ]
-
- c = modelfactories.CollectionMetaFactory.create(is_member=True)
-
- docs = identify_changes(changes, collections=[c], journals=[])
- self.assertEqual(docs[0], "/api/v1/journals/31/")
-
- # def test_identify_journals_given_journals(self):
- # self.assertTrue(True)
@@ -280,3 +280,212 @@ def test_bypass_journals_with_non_member_collections(self):
sync_journals_meta(managerapi_dep=mocker_scieloapi)
self.assertEqual(models.JournalMeta.objects.count(), 0)
+
+
+class ChangesIdentificationTests(mocker.MockerTestCase, TestCase):
+
+ def test_identify_journals_given_collections(self):
+ from utils.tasks import identify_changes
+
+ changes = [
+ {
+ "changed_at": "2013-01-23T15:11:33.409478",
+ "collection_uri": "/api/v1/collections/1/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/31/",
+ "resource_uri": "/api/v1/changes/8/",
+ "seq": 8
+ },
+ {
+ "changed_at": "2013-01-23T15:12:33.409478",
+ "collection_uri": "/api/v1/collections/2/",
+ "event_type": "added",
+ "object_uri": "/api/v1/issues/2840/",
+ "resource_uri": "/api/v1/changes/2/",
+ "seq": 9
+ },
+ ]
+
+ c = modelfactories.CollectionMetaFactory.create()
+
+ docs = identify_changes(changes, collections=[c], journals=[])
+
+ self.assertEqual(len(docs), 1)
+ self.assertEqual(docs[0], "/api/v1/journals/31/")
+
+ def test_identify_journals_given_journals(self):
+ from utils.tasks import identify_changes
+
+ mocker_list_issues = self.mocker.mock()
+ mocker_list_issues(mocker.ANY)
+ self.mocker.result([u'/api/v1/issues/1/'])
+ self.mocker.replay()
+
+ changes = [
+ {
+ "changed_at": "2013-01-23T15:11:33.409478",
+ "collection_uri": "/api/v1/collections/1/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/31/",
+ "resource_uri": "/api/v1/changes/8/",
+ "seq": 8
+ },
+ {
+ "changed_at": "2013-01-23T15:12:33.409478",
+ "collection_uri": "/api/v1/collections/2/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/1/",
+ "resource_uri": "/api/v1/changes/2/",
+ "seq": 9
+ },
+ ]
+
+ j = modelfactories.JournalMetaFactory.create()
+
+ docs = identify_changes(changes,
+ collections=[],
+ journals=[j],
+ list_issues_uri_dep=mocker_list_issues)
+
+ self.assertEqual(len(docs), 1)
+ self.assertEqual(docs[0], "/api/v1/journals/1/")
+
+ def test_identify_journals_given_journals_and_collections(self):
+ from utils.tasks import identify_changes
+
+ mocker_list_issues = self.mocker.mock()
+ mocker_list_issues(mocker.ANY)
+ self.mocker.result([u'/api/v1/issues/1/'])
+ self.mocker.replay()
+
+ changes = [
+ {
+ "changed_at": "2013-01-23T15:11:33.409478",
+ "collection_uri": "/api/v1/collections/1/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/31/",
+ "resource_uri": "/api/v1/changes/8/",
+ "seq": 8
+ },
+ {
+ "changed_at": "2013-01-23T15:12:33.409478",
+ "collection_uri": "/api/v1/collections/2/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/1/",
+ "resource_uri": "/api/v1/changes/2/",
+ "seq": 9
+ },
+ {
+ "changed_at": "2013-01-23T15:13:33.409478",
+ "collection_uri": "/api/v1/collections/2/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/2/",
+ "resource_uri": "/api/v1/changes/3/",
+ "seq": 10
+ },
+ ]
+
+ c = modelfactories.CollectionMetaFactory.create(
+ resource_uri="/api/v1/collections/2/"
+ )
+ c2 = modelfactories.CollectionMetaFactory.create(
+ resource_uri="/api/v1/collections/1/"
+ )
+ j = modelfactories.JournalMetaFactory.create(
+ collection=c,
+ resource_uri="/api/v1/journals/1/"
+ )
+
+ docs = identify_changes(changes,
+ collections=[c2],
+ journals=[j],
+ list_issues_uri_dep=mocker_list_issues)
+
+ self.assertEqual(len(docs), 2)
+ self.assertIn("/api/v1/journals/31/", docs)
+ self.assertIn("/api/v1/journals/1/", docs)
+
+ def test_membership_is_irrelevant(self):
+ from utils.tasks import identify_changes
+
+ mocker_list_issues = self.mocker.mock()
+
+ mocker_list_issues(mocker.ANY)
+ self.mocker.result([u'/api/v1/issues/1/'])
+
+ mocker_list_issues(mocker.ANY)
+ self.mocker.result([])
+
+ self.mocker.replay()
+
+ changes = [
+ {
+ "changed_at": "2013-01-23T15:11:33.409478",
+ "collection_uri": "/api/v1/collections/1/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/1/",
+ "resource_uri": "/api/v1/changes/8/",
+ "seq": 8
+ },
+ {
+ "changed_at": "2013-01-23T15:12:33.409478",
+ "collection_uri": "/api/v1/collections/2/",
+ "event_type": "added",
+ "object_uri": "/api/v1/journals/2/",
+ "resource_uri": "/api/v1/changes/2/",
+ "seq": 9
+ },
+ ]
+
+ j = modelfactories.JournalMetaFactory.create(
+ is_member=True,
+ resource_uri=u'/api/v1/journals/1/')
+ j2 = modelfactories.JournalMetaFactory.create(
+ is_member=False,
+ resource_uri=u'/api/v1/journals/2/')
+
+ docs = identify_changes(changes,
+ collections=[],
+ journals=[j, j2],
+ list_issues_uri_dep=mocker_list_issues)
+
+ self.assertEqual(len(docs), 2)
+
+ def test_identify_issue_changes(self):
+ from utils.tasks import identify_changes
+
+ mocker_list_issues = self.mocker.mock()
+ mocker_list_issues(mocker.ANY)
+ self.mocker.result([u'/api/v1/issues/1/'])
+
+ self.mocker.replay()
+
+ journal_doc = modelfactories.JournalFactory.build()
+ journal_meta = modelfactories.JournalMetaFactory.create()
+
+ changes = [
+ {
+ "changed_at": "2013-01-23T15:11:33.409478",
+ "collection_uri": "/api/v1/collections/1/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/journals/2/",
+ "resource_uri": "/api/v1/changes/8/",
+ "seq": 8
+ },
+ {
+ "changed_at": "2013-01-23T15:12:33.409478",
+ "collection_uri": "/api/v1/collections/1/",
+ "event_type": "updated",
+ "object_uri": "/api/v1/issues/1/",
+ "resource_uri": "/api/v1/changes/2/",
+ "seq": 9
+ },
+ ]
+
+ docs = identify_changes(changes,
+ collections=[],
+ journals=[journal_meta],
+ list_issues_uri_dep=mocker_list_issues)
+
+ self.assertEqual(len(docs), 1)
+ self.assertEqual(docs[0], "/api/v1/issues/1/")

0 comments on commit 26e0490

Please sign in to comment.