Merge pull request #250 from open-contracting/change-uniqueness-rules…

…-for-collections Change uniqueness rules for collections
open-contracting · Dec 20, 2019 · f7df636 · f7df636
2 parents 8229c2e + 97683f6
commit f7df636
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 8 deletions.
diff --git a/ocdskingfisherprocess/database.py b/ocdskingfisherprocess/database.py
@@ -41,14 +41,15 @@ def __init__(self, config):
                                                    nullable=False, default=False),
                                          sa.Column('transform_from_collection_id', sa.Integer,
                                                    sa.ForeignKey("collection.id"), nullable=True),
-                                         sa.Column('transform_type', sa.Text, nullable=True),
+                                         sa.Column('transform_type', sa.Text, nullable=False),
                                          sa.Column('deleted_at', sa.DateTime(timezone=False), nullable=True),
                                          sa.Column('cached_releases_count', sa.Integer, nullable=True),
                                          sa.Column('cached_records_count', sa.Integer, nullable=True),
                                          sa.Column('cached_compiled_releases_count', sa.Integer, nullable=True),
-                                         sa.UniqueConstraint('source_id', 'data_version', 'sample',
-                                                             'transform_from_collection_id', 'transform_type',
-                                                             name='unique_collection_identifiers'),
+                                         sa.Index('unique_collection_identifiers',
+                                                  'source_id', 'data_version', 'sample',
+                                                  unique=True,
+                                                  postgresql_where=sa.text("transform_type = ''")),
                                          sa.Index('collection_transform_from_collection_id_idx',
                                                   'transform_from_collection_id'),
                                          )
@@ -319,7 +320,7 @@ def create_tables(self):
         alembic.config.main(argv=alembicargs)
 
     def get_collection_id(self, source_id, data_version, sample,
-                          transform_from_collection_id=None, transform_type=None):
+                          transform_from_collection_id=None, transform_type=''):
 
         with self.get_engine().begin() as connection:
             s = sa.sql.select([self.collection_table]) \
@@ -334,7 +335,7 @@ def get_collection_id(self, source_id, data_version, sample,
                 return collection['id']
 
     def get_or_create_collection_id(self, source_id, data_version, sample,
-                                    transform_from_collection_id=None, transform_type=None):
+                                    transform_from_collection_id=None, transform_type=''):
 
         collection_id = self.get_collection_id(
             source_id,

diff --git a/...s/maindatabase/migrations/versions/8e3f80979dc9_change_unique_constraint_on_collection.py b/...s/maindatabase/migrations/versions/8e3f80979dc9_change_unique_constraint_on_collection.py
@@ -0,0 +1,43 @@
+"""Change unique constraint on collection
+
+Revision ID: 8e3f80979dc9
+Revises: 3d5fae27a215
+Create Date: 2019-12-18 13:14:56.466907
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8e3f80979dc9'
+down_revision = '3d5fae27a215'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """
+    SELECT source_id, data_version, sample, COUNT(*) FROM collection
+    WHERE transform_type IS NULL or transform_type = ''
+    GROUP BY source_id, data_version, sample
+    HAVING COUNT(*) > 1;
+    """
+    # 0 rows
+
+    op.drop_constraint('unique_collection_identifiers', 'collection')
+    op.create_index('unique_collection_identifiers', 'collection', ['source_id', 'data_version', 'sample'],
+                    unique=True, postgresql_where=sa.text("transform_type = ''"))
+
+    op.execute("UPDATE collection SET transform_type = '' WHERE transform_type IS NULL")
+    op.alter_column('collection', 'transform_type', nullable=False)
+
+
+def downgrade():
+    op.drop_index('unique_collection_identifiers', 'collection')
+    op.create_unique_constraint('unique_collection_identifiers', 'collection', [
+        'source_id', 'data_version', 'sample', 'transform_from_collection_id', 'transform_type',
+    ])
+
+    op.alter_column('collection', 'transform_type', nullable=True)
+    op.execute("UPDATE collection SET transform_type = NULL WHERE transform_type = ''")
diff --git a/ocdskingfisherprocess/models.py b/ocdskingfisherprocess/models.py
@@ -1,6 +1,6 @@
 class CollectionModel:
 
-    def __init__(self, database_id=None, source_id=None, data_version=None, sample=None, transform_type=None,
+    def __init__(self, database_id=None, source_id=None, data_version=None, sample=None, transform_type='',
                  transform_from_collection_id=None, check_data=None, check_older_data_with_schema_version_1_1=None,
                  store_start_at=None, store_end_at=None, deleted_at=None):
         self.database_id = database_id

diff --git a/tests/test_standard_pipeline.py b/tests/test_standard_pipeline.py
@@ -15,7 +15,7 @@ def test_check_on(self):
         collections = self.database.get_all_collections()
         assert 3 == len(collections)
 
-        assert None == collections[0].transform_type # noqa
+        assert '' == collections[0].transform_type # noqa
         assert None == collections[0].transform_from_collection_id # noqa
 
         assert TRANSFORM_TYPE_UPGRADE_1_0_TO_1_1 == collections[1].transform_type