initializedb creates Providers in database. (#849)

Issue #848

atramhasis/scripts/initializedb.py

@@ -1,14 +1,22 @@
+import json
 import os
 import sys
+from datetime import date
+from datetime import datetime
 
 from pyramid.paster import get_appsettings
 from pyramid.paster import setup_logging
 from pyramid.scripts.common import parse_vars
+from skosprovider.providers import VocabularyProvider
+from skosprovider_sqlalchemy import utils as skosprovider_utils
 from skosprovider_sqlalchemy.models import ConceptScheme
 from skosprovider_sqlalchemy.models import Label
-from skosprovider_sqlalchemy.utils import import_provider
 from sqlalchemy import engine_from_config
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import Session
+
+from atramhasis.data.models import ExpandStrategy
+from atramhasis.data.models import IDGenerationStrategy
+from atramhasis.data.models import Provider
 
 
 def usage(argv):
@@ -18,7 +26,13 @@ def usage(argv):
     sys.exit(1)
 
 
-def main(argv=sys.argv):
+def json_serial(obj):
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    raise TypeError(f"Type {obj} not serializable")
+
+
+def initialize_providers(session):
     from fixtures.data import trees, geo
     from fixtures.styles_and_cultures import styles_and_cultures
     from fixtures.materials import materials
@@ -27,17 +41,10 @@ def main(argv=sys.argv):
     from fixtures.periods import periods
     from fixtures.species import species
     from fixtures.bluebirds import bluebirds
-    if len(argv) < 2:
-        usage(argv)
-    config_uri = argv[1]
-    options = parse_vars(argv[2:])
-    setup_logging(config_uri)
-    settings = get_appsettings(config_uri, options=options)
-    engine = engine_from_config(settings, 'sqlalchemy.')
-    db_session = sessionmaker(bind=engine)()
+
     import_provider(
         trees,
-        db_session,
+        session,
         ConceptScheme(
             id=1,
             uri='urn:x-skosprovider:trees',
@@ -49,7 +56,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         geo,
-        db_session,
+        session,
         ConceptScheme(
             id=2,
             uri='urn:x-skosprovider:geo',
@@ -61,7 +68,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         styles_and_cultures,
-        db_session,
+        session,
         ConceptScheme(
             id=3,
             uri='https://id.erfgoed.net/thesauri/stijlen_en_culturen',
@@ -73,7 +80,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         materials,
-        db_session,
+        session,
         ConceptScheme(
             id=4,
             uri='https://id.erfgoed.net/thesauri/materialen',
@@ -85,7 +92,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         eventtypes,
-        db_session,
+        session,
         ConceptScheme(
             id=5,
             uri='https://id.erfgoed.net/thesauri/gebeurtenistypes',
@@ -97,7 +104,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         heritagetypes,
-        db_session,
+        session,
         ConceptScheme(
             id=6,
             uri='https://id.erfgoed.net/thesauri/erfgoedtypes',
@@ -109,7 +116,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         periods,
-        db_session,
+        session,
         ConceptScheme(
             id=7,
             uri='https://id.erfgoed.net/thesauri/dateringen',
@@ -121,7 +128,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         species,
-        db_session,
+        session,
         ConceptScheme(
             id=8,
             uri='https://id.erfgoed.net/thesauri/soorten',
@@ -133,7 +140,7 @@ def main(argv=sys.argv):
     )
     import_provider(
         bluebirds,
-        db_session,
+        session,
         ConceptScheme(
             id=9,
             uri='https://id.bluebirds.org',
@@ -143,8 +150,46 @@ def main(argv=sys.argv):
             ]
         ),
     )
-    db_session.commit()
-    db_session.close()
+
+
+def import_provider(
+    provider: VocabularyProvider, session: Session, conceptscheme: ConceptScheme
+):
+    concept_scheme = skosprovider_utils.import_provider(
+        provider, session, conceptscheme=conceptscheme,
+    )
+
+    if provider.uri_generator:
+        uri_pattern = getattr(provider.uri_generator, 'pattern', None)
+    else:
+        uri_pattern = None
+
+    db_provider = Provider()
+    db_provider.meta = json.loads(json.dumps(provider.metadata, default=json_serial))
+    db_provider.id_generation_strategy = IDGenerationStrategy.NUMERIC
+    db_provider.expand_strategy = ExpandStrategy.RECURSE
+    db_provider.conceptscheme = concept_scheme
+    db_provider.id = provider.get_vocabulary_id()
+    db_provider.uri_pattern = uri_pattern
+    if 'conceptscheme_id' in db_provider.meta:
+        del db_provider.meta['conceptscheme_id']
+
+    session.add(db_provider)
+
+
+def main(argv=sys.argv):
+    if len(argv) < 2:
+        usage(argv)
+
+    config_uri = argv[1]
+    options = parse_vars(argv[2:])
+    setup_logging(config_uri)
+    settings = get_appsettings(config_uri, options=options)
+
+    engine = engine_from_config(settings, 'sqlalchemy.')
+    with Session(engine) as session:
+        initialize_providers(session)
+        session.commit()
     print('--atramhasis-db-initialized--')
 
 

requirements.txt

@@ -4,6 +4,8 @@ pyramid-tm==2.5
 pyramid_rewrite==0.2
 pyramid_openapi3==0.14.3
 openapi-spec-validator==0.4.0  # https://github.com/p1c2u/openapi-core/issues/442
+# need to limit jsonschema until openapi-spec-validator 0.6.0 exists
+jsonschema==4.17.0
 
 # skosprovider
 skosprovider==1.2.0

tests/scripts/test_initializedb.py

@@ -0,0 +1,34 @@
+from sqlalchemy import select
+
+import tests
+from atramhasis.data.models import ExpandStrategy
+from atramhasis.data.models import IDGenerationStrategy
+from atramhasis.data.models import Provider
+from atramhasis.scripts import initializedb as script
+
+
+def setUpModule():
+    tests.setup_db(guarantee_empty=True)
+
+
+class TestMigrateTests(tests.DbTest):
+    def test_initialize_providers(self):
+        script.initialize_providers(self.session)
+
+        expected_ids = [
+            'TREES',
+            'GEOGRAPHY',
+            'STYLE',
+            'MATERIAL',
+            'EVENTTYPE',
+            'HERITAGETYPE',
+            'PERIOD',
+            'SPECIES',
+            'BLUEBIRDS',
+        ]
+        expected_concept_scheme_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+        for db_provider in self.session.execute(select(Provider)).scalars():  # type: Provider
+            self.assertIn(db_provider.conceptscheme.id, expected_concept_scheme_ids)
+            self.assertEqual(db_provider.id_generation_strategy, IDGenerationStrategy.NUMERIC)
+            self.assertEqual(db_provider.expand_strategy, ExpandStrategy.RECURSE)
+            self.assertIn(db_provider.id, expected_ids)