Merge pull request #133 from RockefellerArchiveCenter/bag-info

Saves metadata from bag-info.txt to application

project_electron/orgs/admin.py

@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from django.contrib import admin
-from orgs.models import Organization,User,BAGLogCodes,BAGLog,Archives
+from orgs.models import Organization,User,BAGLogCodes,BAGLog,Archives,BagInfoMetadata
 
 @admin.register(Organization)
 class OrganizationsAdmin(admin.ModelAdmin):
@@ -23,3 +23,7 @@ class BagLogAdmin(admin.ModelAdmin):
 @admin.register(Archives)
 class ArchivesAdmin(admin.ModelAdmin):
 	pass
+
+@admin.register(BagInfoMetadata)
+class BagInfoMetadata(admin.ModelAdmin):
+	pass

project_electron/orgs/migrations/0041_auto_20171119_1216.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2017-11-19 17:16
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orgs', '0040_auto_20171201_1012'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BagInfoMetadata',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('external_identifier', models.CharField(max_length=256)),
+                ('internal_sender_description', models.TextField()),
+                ('title', models.CharField(max_length=256)),
+                ('date_start', models.DateTimeField()),
+                ('date_end', models.DateTimeField()),
+                ('record_type', models.CharField(max_length=30)),
+                ('bagging_date', models.DateTimeField()),
+                ('bag_count', models.CharField(max_length=10)),
+                ('bag_group_identifier', models.CharField(max_length=256)),
+                ('payload_oxum', models.CharField(max_length=20)),
+                ('bagit_profile_identifier', models.URLField()),
+                ('archive', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='orgs.Archives')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='LanguageCode',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('code', models.CharField(max_length=3)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='RecordCreators',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+            ],
+        )
+    ]

project_electron/orgs/migrations/0042_auto_20171119_1237.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2017-11-19 17:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orgs', '0041_auto_20171119_1216'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='baginfometadata',
+            name='language',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='orgs.LanguageCode'),
+        ),
+        migrations.AddField(
+            model_name='baginfometadata',
+            name='record_creators',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='orgs.RecordCreators'),
+        ),
+        migrations.AddField(
+            model_name='baginfometadata',
+            name='source_organization',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='orgs.Organization'),
+        )
+    ]

project_electron/orgs/migrations/0043_auto_20171119_1535.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2017-11-19 20:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orgs', '0042_auto_20171119_1237'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='baginfometadata',
+            name='language',
+        ),
+        migrations.AddField(
+            model_name='baginfometadata',
+            name='language',
+            field=models.ManyToManyField(blank=True, to='orgs.LanguageCode'),
+        ),
+        migrations.RemoveField(
+            model_name='baginfometadata',
+            name='record_creators',
+        ),
+        migrations.AddField(
+            model_name='baginfometadata',
+            name='record_creators',
+            field=models.ManyToManyField(blank=True, to='orgs.RecordCreators'),
+        ),
+    ]

project_electron/orgs/migrations/0044_auto_20171128_1355.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2017-11-28 18:55
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orgs', '0043_auto_20171119_1535'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='baginfometadata',
+            name='archive',
+            field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, to='orgs.Archives'),
+            preserve_default=False,
+        ),
+    ]

project_electron/orgs/models.py

@@ -229,6 +229,12 @@ def get_absolute_url(self):
     class Meta:
         ordering = ['username']
 
+class RecordCreators(models.Model):
+    name = models.CharField(max_length=100)
+
+class LanguageCode(models.Model):
+    code = models.CharField(max_length=3)
+
 class Archives(models.Model):
     machine_file_types = (
         ('ZIP', 'zip'),
@@ -247,7 +253,7 @@ class Archives(models.Model):
 
     organization =          models.ForeignKey(Organization)
     user_uploaded =         models.ForeignKey(User, null=True)
-    machine_file_path =          models.CharField(max_length=100)
+    machine_file_path =     models.CharField(max_length=100)
     machine_file_size =     models.CharField(max_length= 30)
     machine_file_upload_time = models.DateTimeField()
     machine_file_identifier = models.CharField(max_length=255,unique=True)
@@ -298,7 +304,7 @@ def get_bag_failure(self, LAST_ONLY = True):
             'BDIR','EXERR',
             'GBERR', 'RBERR', 
             'MDERR', 'DTERR', 'FSERR',
-            'VIRUS',
+            'VIRUS','BIERR'
         ]
         get_error_obj = BAGLog.objects.filter(archive=self,code__code_short__in=flist)
         if not get_error_obj:
@@ -333,6 +339,63 @@ def setup_save(self, obj):
         elif obj['auto_fail_code'] == 'FSERR':
             self.additional_error_info = 'Bag size ({}) is larger then maximum allow size ({})'.format(obj['file_size'], (settings.TRANSFER_FILESIZE_MAX * 1000))
 
+    def save_mtm_fields(self, cls, field, model_field, metadata):
+        obj_list = []
+        if field in metadata:
+            if type(metadata[field]) is list:
+                for f in metadata[field]:
+                    new_obj = cls.objects.get_or_create(**{model_field: f})[0]
+                    obj_list.append(new_obj)
+            else:
+                new_obj = cls.objects.get_or_create(**{model_field: metadata[field]})[0]
+                obj_list.append(new_obj)
+        return obj_list
+
+    def save_bag_data(self, metadata):
+        if not metadata:
+            return False
+
+        try:
+            creators_list = self.save_mtm_fields(RecordCreators, 'Record_Creators', 'name', metadata)
+            language_list = self.save_mtm_fields(LanguageCode, 'Language', 'code', metadata)
+            item = BagInfoMetadata(
+                archive = self,
+                source_organization = Organization.objects.get(name=metadata['Source_Organization']),
+                external_identifier = metadata.get('External_Identifier', ''),
+                internal_sender_description = metadata.get('Internal_Sender_Description', ''),
+                title = metadata.get('Title', ''),
+                date_start = metadata.get('Date_Start', ''),
+                date_end = metadata.get('Date_End', ''),
+                record_type = metadata.get('Record_Type', ''),
+                bagging_date = metadata.get('Bagging_Date', ''),
+                bag_count = metadata.get('Bag_Count', ''),
+                bag_group_identifier = metadata.get('Bag_Group_Identifier', ''),
+                payload_oxum = metadata.get('Payload_Oxum', ''),
+                bagit_profile_identifier = metadata.get('BagIt_Profile_Identifier', '')
+            )
+            item.save()
+            for c in creators_list:
+                item.record_creators.add(c)
+            for l in language_list:
+                item.language.add(l)
+            item.save()
+        except Exception as e:
+            print e
+            return False
+        else:
+            return True # this at least assumes nothing blew up
+
+    def get_bag_data(self):
+        bag_data = BagInfoMetadata.objects.filter(archive=self.pk).first()
+        excluded_fields = ['id', 'pk', 'archive']
+        field_names = [field.name for field in BagInfoMetadata._meta.get_fields() if field.name not in excluded_fields]
+        values = {}
+        for field_name in field_names:
+            values[field_name] = getattr(bag_data, field_name, '')
+            values[field_name] = getattr(bag_data, field_name, None)
+
+        return values
+
     class Meta:
         ordering = ['machine_file_upload_time']
 
@@ -391,3 +454,20 @@ def log_it(cls, code, archive = None):
 
     class Meta:
         ordering = ['-created_time']
+
+class BagInfoMetadata(models.Model):
+    archive =                       models.ForeignKey(Archives)
+    source_organization =           models.ForeignKey(Organization, blank=True,null=True)
+    external_identifier =           models.CharField(max_length=256)
+    internal_sender_description =   models.TextField()
+    title =                         models.CharField(max_length=256)
+    date_start =                    models.DateTimeField()
+    date_end =                      models.DateTimeField()
+    record_creators =               models.ManyToManyField(RecordCreators, blank=True)
+    record_type =                   models.CharField(max_length=30)
+    language =                      models.ManyToManyField(LanguageCode, blank=True)
+    bagging_date =                  models.DateTimeField()
+    bag_count =                     models.CharField(max_length=10)
+    bag_group_identifier =          models.CharField(max_length=256)
+    payload_oxum =                  models.CharField(max_length=20)
+    bagit_profile_identifier =      models.URLField()

project_electron/transfer_app/lib/bag_checker.py

@@ -7,7 +7,7 @@
 
 from transfer_app.lib import files_helper as FH
 from transfer_app.form import BagInfoForm
-from orgs.models import BAGLog
+from orgs.models import BAGLog, Archives
 
 
 
@@ -48,7 +48,7 @@ def _is_generic_bag(self):
         try:
             self.bag = bagit.Bag(self.archive_path)
             self.bag.validate()
-            
+
         except Exception as e:
             print e
             self.bag_exception = e
@@ -193,6 +193,11 @@ def bag_passed_all(self):
         if not self._has_valid_metadata_file():
             self.ecode = 'MDERR'
             return self.bag_failed()
+
+        if not self.archiveObj.save_bag_data(self.bag_info_data):
+            self.ecode = 'BIERR'
+            return self.bag_failed()
+
         BAGLog.log_it('PBAGP', self.archiveObj)
 
         self.cleanup()
@@ -201,7 +206,9 @@ def bag_passed_all(self):
     def bag_failed(self):
         self.cleanup()
         return False
+
     def cleanup(self):
+        """called on success of failure of bag_checker routine"""
         if self.bag_exception:
             self.archiveObj.additional_error_info = self.bag_exception
 

project_electron/transfer_app/lib/files_helper.py

@@ -66,7 +66,7 @@ def has_files_to_process():
 
                 try:
                     scanresult = virus_checker.scan(file_path)
-                
+
                 except ConnectionError as e:
                     print e
                     BAGLog.log_it('VCON2')
@@ -88,7 +88,7 @@ def has_files_to_process():
                     auto_fail = True
                     auto_fail_code = 'VIRUS'
 
-                    
+
                 else:
                     extension = splitext_(file_path)
                     tar_accepted_ext = ['tar.gz', '.tar']
@@ -476,7 +476,14 @@ def get_fields_from_file(fpath):
 
                 row_search = re.search(":?(\s)?".join(patterns), line)
                 if row_search:
-                    fields[row_search.group('key').replace('-','_').strip()] = row_search.group('val').strip()
+                    key = row_search.group('key').replace('-','_').strip()
+                    val = row_search.group('val').strip()
+                    if key in fields:
+                        listval = [fields[key]]
+                        listval.append(val)
+                        fields[key] = listval
+                    else:
+                        fields[key] = val
     except Exception as e:
         print e
 
@@ -521,4 +528,4 @@ def get_file_contents(f):
     except Exception as e:
         print e
     finally:
-        return data
+        return data

project_electron/transfer_app/templates/transfer_app/transfer_detail.html

@@ -18,12 +18,18 @@
 				<h3 class="box-title">Metadata</h3>
 			</div>
 			<div class="box-body">
+				{% if object.get_bag_data %}
 				<dl class="dl-horizontal">
-					<dt>Key</dt>
-					<dd>Value</dd>
-					<dt>Key</dt>
-					<dd>Value</dd>
+					{% for k,v in object.get_bag_data.items %}
+						{% if v != '' %}
+							<dt>{{k}}</dt>
+							<dd>{{v}}</dd>
+						{% endif %}
+					{% endfor %}
 				</dl>
+				{% else %}
+				<p>No metadata available</p>
+				{% endif %}
 			</div>
 		</div>
 	</div>

project_electron/transfer_app/views.py

@@ -6,7 +6,7 @@
 
 from django.db.models import Sum
 from django.shortcuts import render
-from orgs.models import Archives
+from orgs.models import Archives, BagInfoMetadata
 from orgs.authmixins import LoggedInMixinDefaults
 
 class MainView(LoggedInMixinDefaults, TemplateView):