Merge pull request #3 from open-contracting/date

Change column type from text to datetime

ocdskingfisherprocess/cli/commands/list_collections.py

@@ -19,6 +19,6 @@ def run_command(self, args):
             print("{:5} {:40} {:20} {:5}".format(
                     collection.database_id,
                     collection.source_id[:40],
-                    collection.data_version,
+                    collection.data_version.strftime("%Y-%m-%d %H:%M:%S"),
                     ("Sample" if collection.sample else "Full")
                 ))

ocdskingfisherprocess/database.py

@@ -26,14 +26,13 @@ def __init__(self, config):
         self.collection_table = sa.Table('collection', self.metadata,
                                          sa.Column('id', sa.Integer, primary_key=True),
                                          sa.Column('source_id', sa.Text, nullable=False),
-                                         sa.Column('data_version', sa.Text, nullable=False),
+                                         sa.Column('data_version', sa.DateTime(timezone=False), nullable=False),
                                          sa.Column('store_start_at', sa.DateTime(timezone=False), nullable=False),
                                          sa.Column('store_end_at', sa.DateTime(timezone=False), nullable=True),
                                          sa.Column('sample', sa.Boolean, nullable=False, default=False),
                                          sa.UniqueConstraint('source_id', 'data_version', 'sample'),
                                          )
 
-        # Should be named collection_file but left for backwards compatibility!
         self.collection_file_table = sa.Table('collection_file', self.metadata,
                                               sa.Column('id', sa.Integer, primary_key=True),
                                               sa.Column('collection_id', sa.Integer,

ocdskingfisherprocess/maindatabase/migrations/versions/0177d90da78e_start.py → ocdskingfisherprocess/maindatabase/migrations/versions/45cd673618df_start.py

@@ -1,17 +1,16 @@
 """start
 
-Revision ID: 0177d90da78e
+Revision ID: 45cd673618df
 Revises:
-Create Date: 2018-12-14 08:58:07.868680
+Create Date: 2019-01-08 15:53:26.336666
 
 """
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSONB
 
-
 # revision identifiers, used by Alembic.
-revision = '0177d90da78e'
+revision = '45cd673618df'
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -21,7 +20,7 @@ def upgrade():
     op.create_table('collection',
                     sa.Column('id', sa.Integer, primary_key=True),
                     sa.Column('source_id', sa.Text, nullable=False),
-                    sa.Column('data_version', sa.Text, nullable=False),
+                    sa.Column('data_version', sa.DateTime(timezone=False), nullable=False),
                     sa.Column('store_start_at', sa.DateTime(timezone=False), nullable=False),
                     sa.Column('store_end_at', sa.DateTime(timezone=False), nullable=True),
                     sa.Column('sample', sa.Boolean, nullable=False, default=False),
@@ -55,21 +54,18 @@ def upgrade():
                     )
 
     op.create_table('data',
-
                     sa.Column('id', sa.Integer, primary_key=True),
                     sa.Column('hash_md5', sa.Text, nullable=False, unique=True),
                     sa.Column('data', JSONB, nullable=False),
                     )
 
     op.create_table('package_data',
-
                     sa.Column('id', sa.Integer, primary_key=True),
                     sa.Column('hash_md5', sa.Text, nullable=False, unique=True),
                     sa.Column('data', JSONB, nullable=False),
                     )
 
     op.create_table('release',
-
                     sa.Column('id', sa.Integer, primary_key=True),
                     sa.Column('collection_file_item_id', sa.Integer,
                               sa.ForeignKey("collection_file_item.id"), nullable=False),

ocdskingfisherprocess/util.py

@@ -2,6 +2,7 @@
 import requests
 import json
 import hashlib
+import datetime
 
 RETRY_TIME = 10
 
@@ -129,3 +130,12 @@ def __init__(self, warnings=[], errors=[]):
 #        return content, []
 #    except Exception as e:
 #        error_msg = 'Unable to decode content: %s' % e
+
+
+def parse_string_to_date_time(date_time_string):
+    if " " in date_time_string:
+        # The docs define this format
+        return datetime.datetime.strptime(date_time_string, "%Y-%m-%d %H:%M:%S")
+    else:
+        # But some of our test code uses this format so we'll allow that too.
+        return datetime.datetime.strptime(date_time_string, "%Y-%m-%d-%H-%M-%S")

ocdskingfisherprocess/web/app.py

@@ -4,6 +4,7 @@
 from ocdskingfisherprocess.database import DataBase
 import tempfile
 import os
+from ocdskingfisherprocess.util import parse_string_to_date_time
 
 config = Config()
 config.load_user_config()
@@ -43,9 +44,11 @@ def api_v1_submit_file():
     database = DataBase(config=config)
     store = Store(config=config, database=database)
 
+    collection_data_version = parse_string_to_date_time(request.form.get('collection_data_version'))
+
     store.load_collection(
         request.form.get('collection_source'),
-        request.form.get('collection_data_version'),
+        collection_data_version,
         True if request.form.get('collection_sample', '0') in ['1'] else False,
     )
 
@@ -79,9 +82,11 @@ def api_v1_submit_item():
     database = DataBase(config=config)
     store = Store(config=config, database=database)
 
+    collection_data_version = parse_string_to_date_time(request.form.get('collection_data_version'))
+
     store.load_collection(
         request.form.get('collection_source'),
-        request.form.get('collection_data_version'),
+        collection_data_version,
         True if request.form.get('collection_sample', '0') in ['1'] else False,
     )
 

tests/test_util.py

@@ -0,0 +1,11 @@
+from ocdskingfisherprocess.util import parse_string_to_date_time
+
+
+def test_parse_string_to_date_time_1():
+    date = parse_string_to_date_time("2019-04-01 10:11:12")
+    assert "2019-04-01 10-11-12" == date.strftime("%Y-%m-%d %H-%M-%S")
+
+
+def test_parse_string_to_date_time_2():
+    date = parse_string_to_date_time("2019-04-01-10-11-12")
+    assert "2019-04-01 10-11-12" == date.strftime("%Y-%m-%d %H-%M-%S")