migrate_db and fix squashed migrations

The script migrate_db.py will alter if some sequences couldn't be inserted into
the database due to lack of corresponding voucher specimens.

migrate_db.py

@@ -47,6 +47,7 @@ def __init__(self, xml_string, tables_prefix=None):
         self.table_taxonsets_items = None
         self.table_vouchers_items = None
         self.table_flickr_images_items = []
+        self.list_of_voucher_codes = []
 
     def parse_table_genes(self, xml_string):
         our_data = False
@@ -363,6 +364,8 @@ def import_table_vouchers(self):
             if items_to_flickr is not None:
                 self.table_flickr_images_items += items_to_flickr
 
+            self.list_of_voucher_codes.append(item['code'])
+
     def import_table_sequences(self):
         if self.table_sequences_items is None:
             self.parse_table_sequences(self.dump_string)
@@ -432,10 +435,22 @@ def save_table_sequences_to_db(self):
         if self.table_sequences_items is None:
             self.import_table_sequences()
 
+        seqs_to_insert = []
+        seqs_not_to_insert = []
+        for i in self.table_sequences_items:
+            if i['code_id'] in self.list_of_voucher_codes:
+                seqs_to_insert.append(i)
+            else:
+                seqs_not_to_insert.append(i)
         table = db['public_interface_sequences']
-        table.insert_many(self.table_sequences_items)
+        table.insert_many(seqs_to_insert)
         print("Uploading table `public_interface_sequences`")
 
+        if len(seqs_not_to_insert) > 0:
+            print("Couldn't insert %i sequences due to lack of reference vouchers" % len(seqs_not_to_insert))
+            for i in seqs_not_to_insert:
+                print(i['code_id'], i['gene_code'])
+
     def get_as_tuple(self, string):
         as_tupple = ()
         if string == 'na.gif':
@@ -493,8 +508,8 @@ def convert_to_int(self, string):
 tables_prefix = ''
 parser = ParseXML(dump, tables_prefix)
 
-parser.import_table_sequences()
-parser.save_table_sequences_to_db()
-
 parser.import_table_vouchers()
 parser.save_table_vouchers_to_db()
+
+parser.import_table_sequences()
+parser.save_table_sequences_to_db()

voseq/public_interface/migrations/0001_squashed_0026_auto_20141217_1140.py

@@ -17,6 +17,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('code', models.CharField(unique=True, primary_key=True, max_length=100, help_text='Voucher code.')),
                 ('orden', models.CharField(max_length=100)),
+                ('superfamily', models.CharField(max_length=100, blank=True)),
                 ('family', models.CharField(max_length=100)),
                 ('subfamily', models.CharField(max_length=100)),
                 ('tribe', models.CharField(max_length=100)),
@@ -352,12 +353,6 @@ class Migration(migrations.Migration):
             name='timestamp',
             field=models.DateTimeField(null=True),
         ),
-        migrations.AddField(
-            model_name='Vouchers',
-            name='superfamily',
-            field=models.CharField(null=True, blank=True, max_length=100),
-            preserve_default=True,
-        ),
         migrations.AlterField(
             model_name='Vouchers',
             name='typeSpecies',

voseq/public_interface/migrations/0002_auto_20141218_1315.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('public_interface', '0001_squashed_0026_auto_20141217_1140'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='vouchers',
+            name='superfamily',
+            field=models.CharField(max_length=100, null=True),
+        ),
+    ]

voseq/public_interface/models.py

@@ -105,7 +105,7 @@ class Vouchers(models.Model):
                             help_text="Voucher code.",
                             )
     orden = models.CharField(max_length=100, blank=True, null=True)
-    superfamily = models.CharField(max_length=100, blank=True, null=True)
+    superfamily = models.CharField(max_length=100, null=True)
     family = models.CharField(max_length=100, blank=True, null=True)
     subfamily = models.CharField(max_length=100, blank=True, null=True)
     tribe = models.CharField(max_length=100, blank=True, null=True)