Merge fc2e34c into 5cf431e

.coveragerc

@@ -5,4 +5,5 @@ omit =
     *admin.py
     *strings.py
     *constants.py
+    *models.py
 

requirements/base.txt

@@ -13,3 +13,4 @@ django-rq==1.0.0
 pyyaml
 django-ipware==1.1.6
 django-import-export==0.5.1
+python-crontab==2.2.8

series_tiempo_ar_api/apps/management/actions.py

@@ -2,38 +2,14 @@
 import unicodecsv
 import yaml
 
-from django_rq import job
-from django.utils import timezone
-
 from series_tiempo_ar_api.apps.api.models import Catalog, Dataset
-from .models import DatasetIndexingFile, Node, NodeRegisterFile
-from .strings import DATASET_STATUS, READ_ERROR
+from .models import Node, NodeRegisterFile
+from .strings import DATASET_STATUS
 
 CATALOG_HEADER = u'catalog_id'
 DATASET_ID_HEADER = u'dataset_identifier'
 
 
-@job('indexing')
-def bulk_index(indexing_file_id):
-    indexing_file_model = DatasetIndexingFile.objects.get(id=indexing_file_id)
-    toggler = DatasetIndexableToggler()
-    try:
-        logs_list = toggler.process(indexing_file_model.indexing_file)
-        logs = ''
-        for log in logs_list:
-            logs += log + '\n'
-
-        state = DatasetIndexingFile.PROCESSED
-    except ValueError:
-        logs = READ_ERROR
-        state = DatasetIndexingFile.FAILED
-
-    indexing_file_model.state = state
-    indexing_file_model.logs = logs
-    indexing_file_model.modified = timezone.now()
-    indexing_file_model.save()
-
-
 class DatasetIndexableToggler(object):
 
     def __init__(self):

series_tiempo_ar_api/apps/management/admin.py

@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import time
 from django.contrib import admin
-from .actions import bulk_index, process_node_register_file, confirm_delete
-from .models import DatasetIndexingFile, NodeRegisterFile, Node
+from .actions import process_node_register_file, confirm_delete
+from .tasks import bulk_index, read_datajson
+from .models import DatasetIndexingFile, NodeRegisterFile, Node, IndexingTaskCron, ReadDataJsonTask
 
 
 class BaseRegisterFileAdmin(admin.ModelAdmin):
@@ -70,6 +72,35 @@ def delete_model(self, _, queryset):
                 confirm_delete(node, register_files)
 
 
+class IndexingTaskAdmin(admin.ModelAdmin):
+    list_display = ('__unicode__', 'enabled', 'weekdays_only')
+
+    actions = ('delete_model',)
+
+    def get_actions(self, request):
+        # Borro la acción de borrado default
+        actions = super(IndexingTaskAdmin, self).get_actions(request)
+        if 'delete_selected' in actions:
+            del actions['delete_selected']
+        return actions
+
+    def delete_model(self, request, obj):
+        super(IndexingTaskAdmin, self).delete_model(request, obj)
+        # Actualizo los crons del sistema para reflejar el cambio de modelos
+        obj.update_crontab()
+
+
+class DataJsonAdmin(admin.ModelAdmin):
+    readonly_fields = ('status', 'created', 'finished', 'logs', 'catalogs')
+    list_display = ('__unicode__', 'status')
+
+    def save_model(self, request, obj, form, change):
+        super(DataJsonAdmin, self).save_model(request, obj, form, change)
+        read_datajson.delay(obj)  # Ejecuta indexación
+
+
 admin.site.register(DatasetIndexingFile, DatasetIndexingFileAdmin)
 admin.site.register(NodeRegisterFile, NodeRegisterFileAdmin)
 admin.site.register(Node, NodeAdmin)
+admin.site.register(IndexingTaskCron, IndexingTaskAdmin)
+admin.site.register(ReadDataJsonTask, DataJsonAdmin)

series_tiempo_ar_api/apps/management/management/commands/read_datajson.py

@@ -1,28 +1,14 @@
 #! coding: utf-8
-import logging
-
 from django.core.management import BaseCommand
-from pydatajson import DataJson
-
-from series_tiempo_ar_api.apps.api.indexing import catalog_reader
-from series_tiempo_ar_api.apps.management.models import Node
 
-logger = logging.getLogger(__name__)
-
-READ_ERROR = u"Error en la lectura del catálogo %s: %s"
+from series_tiempo_ar_api.apps.management.models import ReadDataJsonTask
+from series_tiempo_ar_api.apps.management.tasks import read_datajson
 
 
 class Command(BaseCommand):
+    """Comando para ejecutar la indexación manualmente de manera sincrónica,
+    útil para debugging"""
     def handle(self, *args, **options):
-        nodes = Node.objects.filter(indexable=True)
-        for node in nodes:
-            catalog_id = node.catalog_id
-            catalog_url = node.catalog_url
-            try:
-                catalog = DataJson(catalog_url)
-            except (IOError, ValueError, AssertionError) as e:
-                self.stderr.write(READ_ERROR % (catalog_id, e))
-                logging.warn(READ_ERROR, catalog_id, e)
-                continue
-
-            catalog_reader.index_catalog(catalog, catalog_id)
+        task = ReadDataJsonTask()
+        task.save()
+        read_datajson(task, async=False)

series_tiempo_ar_api/apps/management/migrations/0003_indexingtask.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-10 15:05
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0002_auto_20180110_1022'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='IndexingTask',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('day_of_week', models.CharField(choices=[(('MON',), 'Lunes'), (('TUE',), 'Martes'), (('WED',), 'Mi\xe9rcoles'), (('THU',), 'Jueves'), (('FRI',), 'Viernes'), (('SAT',), 'S\xe1bado'), ('SUN', 'Domingo')], max_length=20)),
+                ('time', models.TimeField()),
+                ('enabled', models.BooleanField(default=True)),
+            ],
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0004_auto_20180110_1206.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-10 15:06
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0003_indexingtask'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='indexingtask',
+            name='day_of_week',
+            field=models.CharField(choices=[(('MON',), 'Lunes'), (('TUE',), 'Martes'), (('WED',), 'Mi\xe9rcoles'), (('THU',), 'Jueves'), (('FRI',), 'Viernes'), (('SAT',), 'S\xe1bado'), ('SUN', 'Domingo')], max_length=20, null=True),
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0005_auto_20180110_1208.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-10 15:08
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0004_auto_20180110_1206'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='indexingtask',
+            name='day_of_week',
+            field=models.CharField(blank=True, choices=[(('MON',), 'Lunes'), (('TUE',), 'Martes'), (('WED',), 'Mi\xe9rcoles'), (('THU',), 'Jueves'), (('FRI',), 'Viernes'), (('SAT',), 'S\xe1bado'), ('SUN', 'Domingo')], max_length=20, null=True),
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0006_indexingtask_weekdays_only.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-10 15:12
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0005_auto_20180110_1208'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='indexingtask',
+            name='weekdays_only',
+            field=models.BooleanField(default=False),
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0007_auto_20180110_1455.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-10 17:55
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0006_indexingtask_weekdays_only'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ReadDataJsonTask',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('RUNNING', 'Procesando cat\xe1logos'), ('INDEXING', 'Indexando series'), ('FINISHED', 'Finalizada'), ('ERROR', 'Error')], max_length=20)),
+                ('created', models.DateTimeField()),
+                ('finished', models.DateTimeField(null=True)),
+                ('logs', models.TextField(default='-')),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='indexingtask',
+            name='day_of_week',
+        ),
+        migrations.AlterField(
+            model_name='indexingtask',
+            name='time',
+            field=models.TimeField(help_text='Los segundos ser\xe1n ignorados'),
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0008_readdatajsontask_catalogs.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-11 16:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0023_dataset_present'),
+        ('management', '0007_auto_20180110_1455'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='readdatajsontask',
+            name='catalogs',
+            field=models.ManyToManyField(to='api.Catalog'),
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0009_auto_20180111_1331.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-11 16:31
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0008_readdatajsontask_catalogs'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='readdatajsontask',
+            name='catalogs',
+            field=models.ManyToManyField(to='management.Node'),
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0010_auto_20180111_1412.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-11 17:12
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0009_auto_20180111_1331'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='readdatajsontask',
+            name='catalogs',
+            field=models.ManyToManyField(blank=True, to='management.Node'),
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/0011_auto_20180111_1505.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-11 18:05
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0010_auto_20180111_1412'),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name='IndexingTask',
+            new_name='IndexingTaskCron',
+        ),
+    ]