fix #187: convert uniqkeys to hashes and remove dependence on long te…

…xt fields

alembic/versions/41996f6d756_change_uniques_to_sha1_keys.py

@@ -0,0 +1,62 @@
+"""change uniques to sha1 keys
+
+Revision ID: 41996f6d756
+Revises: 54672c4d904
+Create Date: 2015-03-27 21:22:02.796920
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '41996f6d756'
+down_revision = '54672c4d904'
+
+from alembic import op
+import sqlalchemy as sa
+from pynab.db import Release, windowed_query
+import hashlib
+from sqlalchemy.orm import sessionmaker
+
+def upgrade():
+    # drop duplicate pres
+    conn = op.get_bind()
+
+    conn.execute('''
+        delete from pres
+        where id not in
+        (
+            select min(id)
+            from pres
+            group by requestid, pretime, requestgroup
+        )
+    ''')
+
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('pres_name_key', 'pres', type_='unique')
+    op.create_unique_constraint('pres_uniq', 'pres', ['requestid', 'pretime', 'requestgroup'])
+    op.add_column('releases', sa.Column('uniqhash', sa.String(length=40), nullable=True))
+    op.drop_constraint('releases_name_group_id_posted_key', 'releases', type_='unique')
+    op.create_unique_constraint('releases_uniq', 'releases', ['uniqhash'])
+
+    session = sessionmaker(bind=conn)()
+    # update the hashes
+    q = session.query(Release)
+    for release in windowed_query(q, Release.id, 1000):
+        release.uniqhash = hashlib.sha1(
+            '{}.{}.{}'.format(
+                release.name,
+                release.group_id,
+                release.posted,
+            ).encode('utf-8')).hexdigest()
+
+    session.commit()
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('releases_uniq', 'releases', type_='unique')
+    op.create_unique_constraint('releases_name_group_id_posted_key', 'releases', ['name', 'group_id', 'posted'])
+    op.drop_column('releases', 'uniqhash')
+    op.drop_constraint('pres_uniq', 'pres', type_='unique')
+    op.create_unique_constraint('pres_name_key', 'pres', ['name'])
+    ### end Alembic commands ###

install.py

@@ -86,20 +86,6 @@
             print('Problem inserting data into database: {}'.format(e))
             sys.exit(0)
 
-    print('Copying large pre data into db...')
-    try:
-        nzedb_pre_import.largeNzedbPre()
-    except Exception as e:
-        print('Problem inserting data into database: {}'.format(e))
-        sys.exit(0)
-
-    print('Copying small pre data into db...')
-    try:
-        nzedb_pre_import.nzedbPre()
-    except Exception as e:
-        print('Problem inserting data into database: {}'.format(e))
-        sys.exit(0)
-
     if config.postprocess.get('regex_url'):
         print('Updating regex...')
         pynab.util.update_regex()
@@ -115,6 +101,20 @@
         print(
             'Could not update blacklist. Try the URL in config.py manually - if it doesn\'t work, post an issue on Github.')
 
+    print('Copying large pre data into db...')
+    try:
+        nzedb_pre_import.largeNzedbPre()
+    except Exception as e:
+        print('Problem inserting data into database: {}'.format(e))
+        sys.exit(0)
+
+    print('Copying small pre data into db...')
+    try:
+        nzedb_pre_import.nzedbPre()
+    except Exception as e:
+        print('Problem inserting data into database: {}'.format(e))
+        sys.exit(0)
+
     end = time.time()
 
     print('Install complete in {:.2f}s'.format(end - start))