Permalink
Browse files

Atualização dos periódicos

  • Loading branch information...
1 parent 1cfd1fa commit 1c9cb0f6cb7a957c146ce2c679c593558ccf5e26 @gustavofonseca committed Mar 5, 2013
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding model 'Sync'
+ db.create_table(u'catalog_sync', (
+ (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('started_at', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+ ('ended_at', self.gf('django.db.models.fields.DateTimeField')(null=True)),
+ ('last_seq', self.gf('django.db.models.fields.IntegerField')(default=0)),
+ ('status', self.gf('django.db.models.fields.CharField')(default='started', max_length=32)),
+ ))
+ db.send_create_signal(u'catalog', ['Sync'])
+
+
+ def backwards(self, orm):
+ # Deleting model 'Sync'
+ db.delete_table(u'catalog_sync')
+
+
+ models = {
+ u'catalog.collectionmeta': {
+ 'Meta': {'object_name': 'CollectionMeta'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_member': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'name_slug': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'resource_uri': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+ },
+ u'catalog.journalmeta': {
+ 'Meta': {'object_name': 'JournalMeta'},
+ 'collection': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'journals'", 'to': u"orm['catalog.CollectionMeta']"}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_member': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'resource_uri': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+ },
+ u'catalog.sync': {
+ 'Meta': {'ordering': "['-ended_at']", 'object_name': 'Sync'},
+ 'ended_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'last_seq': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'started_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'started'", 'max_length': '32'})
+ }
+ }
+
+ complete_apps = ['catalog']
View
@@ -1,6 +1,13 @@
from django.db import models
+SYNC_STATUS_CHOICES = [
+ ('started', 'started'),
+ ('finished', 'finished'),
+ ('failed', 'failed'),
+]
+
+
class CustomMetaManager(models.Manager):
def members(self):
qset = self.get_query_set().filter(is_member=True)
@@ -66,3 +73,7 @@ class Sync(models.Model):
started_at = models.DateTimeField(auto_now_add=True)
ended_at = models.DateTimeField(null=True)
last_seq = models.IntegerField(default=0)
+ status = models.CharField(choices=SYNC_STATUS_CHOICES, max_length=32, default='started')
+
+ class Meta:
+ ordering = ['-ended_at']
View
@@ -76,13 +76,7 @@ def _what_have_changed(since=0, managerapi_dep=datacollector.SciELOManagerAPI):
changes = changes_list.filter(collections=full_collections,
journals=journals_a_la_carte)
- journals = [ch.resource_id for ch in changes.show('journals', unique=True)]
- # issues = changes.show('issues', unique=True)
-
- return {
- 'journals': scielo_api.get_journals(*journals),
- # 'issues': changes.show('issues', unique=True),
- }
+ return changes
def _list_issues_uri(journal_meta, journal_dep=mongomodels.Journal):
@@ -146,3 +140,12 @@ def identify_changes(changes,
continue
return {'journals': list(changed_journals), 'issues': list(changed_issues)}
+
+
+def get_last_seq():
+ last_sync = models.Sync.objects.all()[0]
+
+ if last_sync:
+ return last_sync.last_seq
+ else:
+ return 0
@@ -165,7 +165,8 @@ def _list_issues_uri(journal_meta, journal_dep=mongomodels.Journal):
class ChangeListIterator(object):
def __init__(self, data):
- self._data = sorted(copy.deepcopy(data), key=lambda x: x.seq)
+ # self._data = sorted(copy.deepcopy(data), key=lambda x: x.seq)
+ self._data = data
self._index = -1
def __iter__(self):
@@ -207,6 +208,7 @@ def __init__(self, data, list_issues_uri_dep=_list_issues_uri):
raise TypeError()
self.list_issues_uri = list_issues_uri_dep
+ self._changes.sort(key=lambda x: x.seq)
def filter(self, collections=None, journals=None):
"""
@@ -276,6 +278,10 @@ def show(self, endpoint, unique=False):
else:
continue
+ @property
+ def last_seq(self):
+ return self._changes[-1].seq
+
class Change(object):
View
@@ -1,4 +1,7 @@
+from datetime import datetime
+
from django.conf import settings
+from django.db import transaction
from celery import task
from .sync import datacollector
@@ -22,15 +25,36 @@ def build_catalog():
marreta = dataloader.Marreta(settings=settings)
marreta.rebuild_collection('journals', transformed_data)
+ models.Sync.objects.create(ended_at=datetime.now(),
+ last_seq=60, status='finished')
+
-def update_catalog():
+def update_catalog(managerapi_dep=datacollector.SciELOManagerAPI):
+ scielo_api = managerapi_dep(settings=settings)
journal_ppl = functions.make_journal_pipeline()
- data = functions._what_have_changed()
- transformed_data = journal_ppl.run(data['journals'])
+ with transaction.commit_on_success():
+ sync = models.Sync.objects.create()
- marreta = dataloader.Marreta(settings=settings)
- marreta.update_collection('journals', transformed_data)
+ changes = functions._what_have_changed(since=functions.get_last_seq())
+ changed_journals = changes.show('journals', unique=True)
+ # changed_issues = changes.show('issues', unique=True)
+
+ changed_journals_ids = [ch.resource_id for ch in changed_journals]
+ # changed_issues_ids = [ch.resource_id for ch in changed_issues]
+
+ journals_data = scielo_api.get_journals(*changed_journals_ids)
+ # issues_data = scielo_api.get_issues(*changed_issues_ids)
+
+ transformed_journals_data = journal_ppl.run(journals_data)
+
+ marreta = dataloader.Marreta(settings=settings)
+ marreta.update_collection('journals', transformed_journals_data)
+
+ sync.last_seq = changes.last_seq
+ sync.status = 'finished'
+ sync.ended_at = datetime.now()
+ sync.save()
def sync_collections_meta(managerapi_dep=datacollector.SciELOManagerAPI):

0 comments on commit 1c9cb0f

Please sign in to comment.