Agrego opcion para cargar nodos desde un yml

series_tiempo_ar_api/apps/management/admin.py

@@ -1,38 +1,72 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import yaml
 from django.contrib import admin
 from .bulk_index import bulk_index
-from .models import DatasetIndexingFile, Node
+from .models import DatasetIndexingFile, NodeRegisterFile, Node
 
 
-class DatasetsIndexingFileAdmin(admin.ModelAdmin):
+class BaseRegisterFileAdmin(admin.ModelAdmin):
     actions = ['run']
 
     readonly_fields = ('created', 'modified', 'state', 'logs')
 
+    def run(self, _, queryset):
+        raise NotImplementedError
+    run.short_description = 'Ejecutar'
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super(BaseRegisterFileAdmin, self).get_form(request, obj, **kwargs)
+        form.base_fields['uploader'].initial = request.user
+        return form
+
+    def save_form(self, request, form, change):
+        return super(BaseRegisterFileAdmin, self).save_form(request, form, change)
+
+
+class DatasetIndexingFileAdmin(BaseRegisterFileAdmin):
     def run(self, _, queryset):
         for model in queryset:
             model.state = DatasetIndexingFile.state = DatasetIndexingFile.PROCESSING
             model.logs = u'-'
             model.save()
             bulk_index.delay(model.id)
 
-    run.short_description = 'Ejecutar'
 
-    def get_form(self, request, obj=None, **kwargs):
-        form = super(DatasetsIndexingFileAdmin, self).get_form(request, obj, **kwargs)
-        form.base_fields['uploader'].initial = request.user
-        return form
+class NodeRegisterFileAdmin(BaseRegisterFileAdmin):
+    def run(self, _, queryset):
+        for model in queryset:
+            model.state = NodeRegisterFile.state = NodeRegisterFile.PROCESSING
+            model.logs = u'-'
+            model.save()
+            process_node_register_file(model)
 
-    def save_form(self, request, form, change):
-        return super(DatasetsIndexingFileAdmin, self).save_form(request, form, change)
+
+def process_node_register_file(model):
+    indexing_file = model.indexing_file
+    yml = indexing_file.read()
+    nodes = yaml.load(yml)
+    for node, values in nodes.items():
+        if bool(values['federado']) is True and values.get('formato') == 'json':
+            node_model, _ = Node.objects.get_or_create(catalog_id=node,
+                                                       catalog_url=values['url'],
+                                                       indexable=True)
+            node_model.save()
+
+    model.status = NodeRegisterFile.PROCESSED
 
 
 class NodeAdmin(admin.ModelAdmin):
 
     list_display = ('catalog_id', 'indexable')
-    actions =  ('make_indexable', 'make_unindexable')
+    actions = ('delete_model', 'run_indexing', 'make_indexable', 'make_unindexable')
+
+    def get_actions(self, request):
+        actions = super(NodeAdmin, self).get_actions(request)
+        if 'delete_selected' in actions:
+            del actions['delete_selected']
+        return actions
 
     def make_unindexable(self, _, queryset):
         queryset.update(indexable=False)
@@ -42,6 +76,18 @@ def make_indexable(self, _, queryset):
         queryset.update(indexable=True)
     make_indexable.short_description = 'Marcar como indexable'
 
+    def delete_model(self, _, queryset):
+        for node in queryset:
+            if node.indexable:
+                for register_file in NodeRegisterFile.objects.all():
+                    indexing_file = register_file.indexing_file
+                    yml = indexing_file.read()
+                    nodes = yaml.load(yml)
+                    if node.catalog_id not in nodes or not nodes[node.catalog_id].get('federado'):
+                        node.delete()
+                        break
+
 
-admin.site.register(DatasetIndexingFile, DatasetsIndexingFileAdmin)
-admin.site.register(Node, NodeAdmin)
+admin.site.register(DatasetIndexingFile, DatasetIndexingFileAdmin)
+admin.site.register(NodeRegisterFile, NodeRegisterFileAdmin)
+admin.site.register(Node, NodeAdmin)

series_tiempo_ar_api/apps/management/migrations/0005_auto_20180108_1430.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-01-08 17:30
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('management', '0004_node'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BaseRegisterFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField()),
+                ('modified', models.DateTimeField(null=True)),
+                ('indexing_file', models.FileField(upload_to='register_files/')),
+                ('state', models.CharField(choices=[('UPLOADED', 'Cargado'), ('PROCESSING', 'Procesando'), ('PROCESSED', 'Procesado'), ('FAILED', 'Error')], max_length=20)),
+                ('logs', models.TextField(default='-')),
+            ],
+        ),
+        migrations.AlterModelOptions(
+            name='node',
+            options={},
+        ),
+        migrations.RemoveField(
+            model_name='datasetindexingfile',
+            name='created',
+        ),
+        migrations.RemoveField(
+            model_name='datasetindexingfile',
+            name='id',
+        ),
+        migrations.RemoveField(
+            model_name='datasetindexingfile',
+            name='indexing_file',
+        ),
+        migrations.RemoveField(
+            model_name='datasetindexingfile',
+            name='logs',
+        ),
+        migrations.RemoveField(
+            model_name='datasetindexingfile',
+            name='modified',
+        ),
+        migrations.RemoveField(
+            model_name='datasetindexingfile',
+            name='state',
+        ),
+        migrations.RemoveField(
+            model_name='datasetindexingfile',
+            name='uploader',
+        ),
+        migrations.CreateModel(
+            name='NodeRegisterFile',
+            fields=[
+                ('baseregisterfile_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='management.BaseRegisterFile')),
+            ],
+            bases=('management.baseregisterfile',),
+        ),
+        migrations.AddField(
+            model_name='baseregisterfile',
+            name='uploader',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='datasetindexingfile',
+            name='baseregisterfile_ptr',
+            field=models.OneToOneField(auto_created=True, default=1, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='management.BaseRegisterFile'),
+            preserve_default=False,
+        ),
+    ]

series_tiempo_ar_api/apps/management/models.py

@@ -5,7 +5,8 @@
 from django.utils import timezone
 
 
-class DatasetIndexingFile(models.Model):
+class BaseRegisterFile(models.Model):
+
     UPLOADED = "UPLOADED"
     PROCESSING = "PROCESSING"
     PROCESSED = "PROCESSED"
@@ -20,7 +21,7 @@ class DatasetIndexingFile(models.Model):
 
     created = models.DateTimeField()
     modified = models.DateTimeField(null=True)
-    indexing_file = models.FileField(upload_to='datasets_indexing_files/')
+    indexing_file = models.FileField(upload_to='register_files/')
     uploader = models.ForeignKey(User)
     state = models.CharField(max_length=20, choices=STATE_CHOICES)
     logs = models.TextField(default=u'-')
@@ -31,8 +32,10 @@ def save(self, force_insert=False, force_update=False, using=None,
             self.created = timezone.now()
             self.state = self.UPLOADED
 
-        super(DatasetIndexingFile, self).save(force_insert, force_update, using, update_fields)
+        super(BaseRegisterFile, self).save(force_insert, force_update, using, update_fields)
+
 
+class DatasetIndexingFile(BaseRegisterFile):
     def __unicode__(self):
         return "Indexing file: {}".format(self.created)
 
@@ -45,3 +48,8 @@ class Node(models.Model):
 
     def __unicode__(self):
         return self.catalog_id
+
+
+class NodeRegisterFile(BaseRegisterFile):
+    def __unicode__(self):
+        return "Node register file: {}".format(self.created)