[#526] Refactor tests for the new setup and add some more

opentrials/opentrials#526

schema_utils.py

@@ -6,7 +6,6 @@
 
 import io
 import sys
-import getopt
 from sqlalchemy import create_engine, MetaData
 from sqlalchemy.orm import sessionmaker
 

tests/conftest.py

@@ -6,57 +6,25 @@
 
 import pytest
 import dataset
-from sqlalchemy import create_engine, MetaData
+from sqlalchemy import MetaData
 from sqlalchemy.orm import sessionmaker
 import processors.base.config as config
 
 # Make fixtures available to all tests
 
-from tests.fixtures.files import file_fixture
-from tests.fixtures.trials import trial
-from tests.fixtures.sources import nct_source, fda_source
-from tests.fixtures.fda_approvals import fda_approval
-from tests.fixtures.fda_applications import fda_application
-from tests.fixtures.organizations import organization
+from tests.fixtures.api.files import file_fixture, fda_file
+from tests.fixtures.api.trials import trial
+from tests.fixtures.api.sources import nct_source, fda_source, euctr_source
+from tests.fixtures.api.fda_approvals import fda_approval
+from tests.fixtures.api.fda_applications import fda_application
+from tests.fixtures.api.organizations import organization
+from tests.fixtures.api.records import record
+from tests.fixtures.api.documents import fda_document
 
+from tests.fixtures.warehouse.cochrane_reviews import cochrane_review
 
-def teardown_database(engine):
-    metadata = MetaData(bind=engine)
-    metadata.reflect()
-    metadata.drop_all()
-
-
-def create_test_database(source_database_url, test_database_url):
-    engine = create_engine(source_database_url)
-    test_engine = create_engine(test_database_url)
-    metadata = MetaData()
-    metadata.reflect(engine)
-    metadata.create_all(test_engine)
-    return test_engine
-
-
-@pytest.fixture(scope='session')
-def setup_test_databases(request):
-    """Create test databases from the schema of source databases.
-        The databases are created when the first test uses them and are dropped
-         at the end of each test session.
-
-    Returns:
-        a tuple of SQLAlchemy engines for the test databases
-    """
-
-    test_warehouse = create_test_database(config.WAREHOUSE_URL, config.TEST_WAREHOUSE_URL)
-    test_api = create_test_database(config.DATABASE_URL, config.TEST_DATABASE_URL)
-    def teardown():
-        teardown_database(test_warehouse)
-        teardown_database(test_api)
-
-    request.addfinalizer(teardown)
-    return (test_warehouse, test_api)
-
-
-@pytest.fixture(scope='function')
-def conn(request, setup_test_databases):
+@pytest.fixture
+def conn(request):
     """Create connection dict for the test databases.
         New sessions are created for each test and are closed at the end of the test.
 
@@ -65,7 +33,6 @@ def conn(request, setup_test_databases):
          a `dataset.Database()` instance
     """
 
-    test_warehouse, test_api = setup_test_databases
     conn = {}
     conn['database'] = dataset.connect(config.TEST_DATABASE_URL)
     conn['warehouse'] = dataset.connect(config.TEST_WAREHOUSE_URL)
@@ -75,8 +42,20 @@ def conn(request, setup_test_databases):
     WarehouseSession = sessionmaker(bind=conn['warehouse'].engine)
     warehouse_session = WarehouseSession()
     def teardown():
+        truncate_database(conn['database'].engine)
+        truncate_database(conn['warehouse'].engine)
         api_session.close()
         warehouse_session.close()
 
     request.addfinalizer(teardown)
     return conn
+
+def truncate_database(engine):
+    metadata = MetaData(bind=engine)
+    metadata.reflect()
+
+    connection = engine.connect()
+    transaction = connection.begin()
+    for table in reversed(metadata.sorted_tables):
+        connection.execute(table.delete())
+    transaction.commit()

tests/fixtures/api/documents.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import uuid
+import pytest
+
+@pytest.fixture
+def fda_document(conn, fda_approval, fda_file, fda_source):
+    document = {
+        'id': uuid.uuid1().hex,
+        'source_id': fda_source,
+        'name': 'Printed Labeling',
+        'type': 'other',
+        'fda_approval_id': fda_approval,
+        'file_id': fda_file,
+        'source_url': None,
+    }
+    document_id = conn['database']['documents'].insert(document)
+    return document_id

tests/fixtures/fda_applications.py → tests/fixtures/api/fda_applications.py

@@ -7,7 +7,7 @@
 import pytest
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture
 def fda_application(conn, organization):
     fda_application = {
         'id': 'ANDA018659',

tests/fixtures/fda_approvals.py → tests/fixtures/api/fda_approvals.py

@@ -7,7 +7,7 @@
 import pytest
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture
 def fda_approval(conn, fda_application):
     fda_approval = {
         'id': ('%s-002' % fda_application),

tests/fixtures/api/files.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import pytest
+import uuid
+
+
+@pytest.fixture
+def file_fixture(conn):
+    sha1 = uuid.uuid1().hex
+    file_record = {
+        'id': uuid.uuid1().hex,
+        'sha1': sha1,
+        'source_url': ('http://example.org/file_%s.pdf' % sha1),
+    }
+    file_id = conn['database']['files'].insert(file_record)
+    return file_id
+
+
+@pytest.fixture
+def fda_file(conn):
+    sha1 = uuid.uuid1().hex
+    file_record = {
+        'id':  uuid.uuid1().hex,
+        'sha1': sha1,
+        'source_url': ('http://datastore.opentrials.net/documents/fda/file_%s.pdf' % sha1),
+        'documentcloud_id': 3154193,
+        'pages': [],
+    }
+    file_id = conn['database']['files'].insert(file_record)
+    return file_id

tests/fixtures/organizations.py → tests/fixtures/api/organizations.py

@@ -8,7 +8,7 @@
 import uuid
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture
 def organization(conn, fda_source):
     organisation = {
         'id': uuid.uuid1().hex,

tests/fixtures/api/records.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import pytest
+import uuid
+
+@pytest.fixture
+def record(conn, nct_source):
+    record = {
+        'id': uuid.uuid1().hex,
+        'identifiers': {'nct': 'NCT00212927'},
+        'registration_date': '2005-09-13',
+        'public_title': 'Continuity of Care and Outcomes After Discharge From Hospital',
+        'brief_summary': None,
+        'scientific_title': 'Patient Outcomes After Discharge From Hospital',
+        'description': 'See above',
+        'recruitment_status': 'not_recruiting',
+        'eligibility_criteria': None,
+        'first_enrollment_date': '2002-10-01',
+        'study_type': 'Observational',
+        'study_design': 'Prospective',
+        'study_phase': None,
+        'primary_outcomes': None,
+        'secondary_outcomes': None,
+        'has_published_results': False,
+        'gender': 'both',
+        'status': 'complete',
+        'completion_date': '2006-10-13',
+        'source_id': nct_source,
+        'source_url': 'https://clinicaltrials.gov/ct2/show/NCT00212927',
+        'source_data': {},
+        'trial_id': None,
+    }
+    record_id = conn['database']['records'].insert(record)
+    return record_id

tests/fixtures/sources.py → tests/fixtures/api/sources.py

@@ -7,7 +7,7 @@
 import pytest
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture
 def nct_source(conn):
     source = {
         'id': 'nct',
@@ -20,7 +20,7 @@ def nct_source(conn):
     return source_id
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture
 def fda_source(conn):
     source = {
         'id': 'fda',
@@ -30,3 +30,16 @@ def fda_source(conn):
     }
     source_id = conn['database']['sources'].insert(source)
     return source_id
+
+
+@pytest.fixture
+def euctr_source(conn):
+    source = {
+        'id': 'euctr',
+        'name': 'EU Clinical Trials Register',
+        'type': 'register',
+        'source_url': 'https://www.clinicaltrialsregister.eu',
+        'terms_and_conditions_url': 'https://www.clinicaltrialsregister.eu/disclaimer.html',
+      }
+    source_id = conn['database']['sources'].insert(source)
+    return source_id

tests/fixtures/trials.py → tests/fixtures/api/trials.py

@@ -7,24 +7,24 @@
 import pytest
 import uuid
 
-@pytest.fixture(scope='function')
+@pytest.fixture
 def trial(conn, nct_source):
     trial = {
         'id': uuid.uuid1().hex,
         'identifiers': {'nct': 'NCT00212927'},
         'registration_date': '2005-09-13',
         'public_title': 'Continuity of Care and Outcomes After Discharge From Hospital',
-        'brief_summary': '',
+        'brief_summary': None,
         'scientific_title': 'Patient Outcomes After Discharge From Hospital',
         'description': 'See above',
         'recruitment_status': 'not_recruiting',
-        'eligibility_criteria': '',
+        'eligibility_criteria': None,
         'first_enrollment_date': '2002-10-01',
         'study_type': 'Observational',
         'study_design': 'Prospective',
-        'study_phase': '',
-        'primary_outcomes': '',
-        'secondary_outcomes': '',
+        'study_phase': None,
+        'primary_outcomes': None,
+        'secondary_outcomes': None,
         'has_published_results': False,
         'gender': 'both',
         'status': 'complete',

tests/fixtures/warehouse/cochrane_reviews.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import uuid
+import pytest
+
+@pytest.fixture
+def cochrane_review(conn):
+    review = {
+        'id': uuid.uuid1().hex,
+        'study_type': 'MIX',
+        'file_name': 'Sulpiride for schizophrenia [v9.0-For publication].rm5',
+        'meta_source': 'http://datastore.opentrials.net/uploads/d7823d80-6f81-11e6-87af-931e370d0cf8/cochrane_schizophrenia_reviews.zip',
+        'doi_id': '10.1002/14651858.CD001162',
+        'study_id': 'STD-Soni-1990',
+        'robs': [
+            {
+                'result': 'YES',
+                'rob_id': 'QIT-02',
+                'group_id': '',
+                'modified': '',
+                'rob_name': 'Allocation concealment?',
+                'study_id': 'STD-Soni-1990',
+                'group_name': '', 'rob_description':
+                'Was allocation adequately concealed?',
+                'result_description': 'A - Adequate',
+            },
+        ],
+        'refs': [
+            {
+                'no': '',
+                'pg': '233-8',
+                'vl': '5', 'type':
+                'JOURNAL_ARTICLE',
+                'year': '1990',
+                'title': 'Sulpiride in negative schizophrenia - a placebo-controlled double-blind assessment',
+                'source': 'Human Psychopharmacology Clinical and Experimental',
+                'authors': 'Soni SD, Mallik A, Schiff AA',
+                'country': '',
+                'identifiers': [],
+            },
+        ],
+    }
+    review_id = conn['warehouse']['cochrane_reviews'].insert(review)
+    return review_id