From 700a721fd38f2cad8af3cb0cfeb888ffad7ebc35 Mon Sep 17 00:00:00 2001 From: Wim De Clercq Date: Mon, 10 Jul 2023 13:50:45 +0200 Subject: [PATCH] initializedb creates Providers in database. (#849) Issue #848 --- atramhasis/scripts/initializedb.py | 89 ++++++++++++++++++++++-------- requirements.txt | 2 + tests/scripts/test_initializedb.py | 34 ++++++++++++ 3 files changed, 103 insertions(+), 22 deletions(-) create mode 100644 tests/scripts/test_initializedb.py diff --git a/atramhasis/scripts/initializedb.py b/atramhasis/scripts/initializedb.py index 52845d35..2c0e911b 100644 --- a/atramhasis/scripts/initializedb.py +++ b/atramhasis/scripts/initializedb.py @@ -1,14 +1,22 @@ +import json import os import sys +from datetime import date +from datetime import datetime from pyramid.paster import get_appsettings from pyramid.paster import setup_logging from pyramid.scripts.common import parse_vars +from skosprovider.providers import VocabularyProvider +from skosprovider_sqlalchemy import utils as skosprovider_utils from skosprovider_sqlalchemy.models import ConceptScheme from skosprovider_sqlalchemy.models import Label -from skosprovider_sqlalchemy.utils import import_provider from sqlalchemy import engine_from_config -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import Session + +from atramhasis.data.models import ExpandStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider def usage(argv): @@ -18,7 +26,13 @@ def usage(argv): sys.exit(1) -def main(argv=sys.argv): +def json_serial(obj): + if isinstance(obj, (datetime, date)): + return obj.isoformat() + raise TypeError(f"Type {obj} not serializable") + + +def initialize_providers(session): from fixtures.data import trees, geo from fixtures.styles_and_cultures import styles_and_cultures from fixtures.materials import materials @@ -27,17 +41,10 @@ def main(argv=sys.argv): from fixtures.periods import periods from fixtures.species import species from fixtures.bluebirds import bluebirds - if len(argv) < 2: - usage(argv) - config_uri = argv[1] - options = parse_vars(argv[2:]) - setup_logging(config_uri) - settings = get_appsettings(config_uri, options=options) - engine = engine_from_config(settings, 'sqlalchemy.') - db_session = sessionmaker(bind=engine)() + import_provider( trees, - db_session, + session, ConceptScheme( id=1, uri='urn:x-skosprovider:trees', @@ -49,7 +56,7 @@ def main(argv=sys.argv): ) import_provider( geo, - db_session, + session, ConceptScheme( id=2, uri='urn:x-skosprovider:geo', @@ -61,7 +68,7 @@ def main(argv=sys.argv): ) import_provider( styles_and_cultures, - db_session, + session, ConceptScheme( id=3, uri='https://id.erfgoed.net/thesauri/stijlen_en_culturen', @@ -73,7 +80,7 @@ def main(argv=sys.argv): ) import_provider( materials, - db_session, + session, ConceptScheme( id=4, uri='https://id.erfgoed.net/thesauri/materialen', @@ -85,7 +92,7 @@ def main(argv=sys.argv): ) import_provider( eventtypes, - db_session, + session, ConceptScheme( id=5, uri='https://id.erfgoed.net/thesauri/gebeurtenistypes', @@ -97,7 +104,7 @@ def main(argv=sys.argv): ) import_provider( heritagetypes, - db_session, + session, ConceptScheme( id=6, uri='https://id.erfgoed.net/thesauri/erfgoedtypes', @@ -109,7 +116,7 @@ def main(argv=sys.argv): ) import_provider( periods, - db_session, + session, ConceptScheme( id=7, uri='https://id.erfgoed.net/thesauri/dateringen', @@ -121,7 +128,7 @@ def main(argv=sys.argv): ) import_provider( species, - db_session, + session, ConceptScheme( id=8, uri='https://id.erfgoed.net/thesauri/soorten', @@ -133,7 +140,7 @@ def main(argv=sys.argv): ) import_provider( bluebirds, - db_session, + session, ConceptScheme( id=9, uri='https://id.bluebirds.org', @@ -143,8 +150,46 @@ def main(argv=sys.argv): ] ), ) - db_session.commit() - db_session.close() + + +def import_provider( + provider: VocabularyProvider, session: Session, conceptscheme: ConceptScheme +): + concept_scheme = skosprovider_utils.import_provider( + provider, session, conceptscheme=conceptscheme, + ) + + if provider.uri_generator: + uri_pattern = getattr(provider.uri_generator, 'pattern', None) + else: + uri_pattern = None + + db_provider = Provider() + db_provider.meta = json.loads(json.dumps(provider.metadata, default=json_serial)) + db_provider.id_generation_strategy = IDGenerationStrategy.NUMERIC + db_provider.expand_strategy = ExpandStrategy.RECURSE + db_provider.conceptscheme = concept_scheme + db_provider.id = provider.get_vocabulary_id() + db_provider.uri_pattern = uri_pattern + if 'conceptscheme_id' in db_provider.meta: + del db_provider.meta['conceptscheme_id'] + + session.add(db_provider) + + +def main(argv=sys.argv): + if len(argv) < 2: + usage(argv) + + config_uri = argv[1] + options = parse_vars(argv[2:]) + setup_logging(config_uri) + settings = get_appsettings(config_uri, options=options) + + engine = engine_from_config(settings, 'sqlalchemy.') + with Session(engine) as session: + initialize_providers(session) + session.commit() print('--atramhasis-db-initialized--') diff --git a/requirements.txt b/requirements.txt index 1d798933..bc32b68e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ pyramid-tm==2.5 pyramid_rewrite==0.2 pyramid_openapi3==0.14.3 openapi-spec-validator==0.4.0 # https://github.com/p1c2u/openapi-core/issues/442 +# need to limit jsonschema until openapi-spec-validator 0.6.0 exists +jsonschema==4.17.0 # skosprovider skosprovider==1.2.0 diff --git a/tests/scripts/test_initializedb.py b/tests/scripts/test_initializedb.py new file mode 100644 index 00000000..b2f6230e --- /dev/null +++ b/tests/scripts/test_initializedb.py @@ -0,0 +1,34 @@ +from sqlalchemy import select + +import tests +from atramhasis.data.models import ExpandStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider +from atramhasis.scripts import initializedb as script + + +def setUpModule(): + tests.setup_db(guarantee_empty=True) + + +class TestMigrateTests(tests.DbTest): + def test_initialize_providers(self): + script.initialize_providers(self.session) + + expected_ids = [ + 'TREES', + 'GEOGRAPHY', + 'STYLE', + 'MATERIAL', + 'EVENTTYPE', + 'HERITAGETYPE', + 'PERIOD', + 'SPECIES', + 'BLUEBIRDS', + ] + expected_concept_scheme_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9] + for db_provider in self.session.execute(select(Provider)).scalars(): # type: Provider + self.assertIn(db_provider.conceptscheme.id, expected_concept_scheme_ids) + self.assertEqual(db_provider.id_generation_strategy, IDGenerationStrategy.NUMERIC) + self.assertEqual(db_provider.expand_strategy, ExpandStrategy.RECURSE) + self.assertIn(db_provider.id, expected_ids)