From 018914239b99071b5756a652afe232bd0765d435 Mon Sep 17 00:00:00 2001 From: Wim De Clercq Date: Tue, 2 May 2023 11:04:51 +0200 Subject: [PATCH] Add Create, Update, Delete provider. (#744, PR:#812) Issue #744 --- .../b2a7d7614973_add_provider_table.py | 40 +++ atramhasis/data/datamanagers.py | 28 ++- atramhasis/data/models.py | 76 +++++- atramhasis/json_processors/__init__.py | 0 atramhasis/json_processors/provider.py | 45 ++++ atramhasis/mappers.py | 37 +++ atramhasis/openapi.yaml | 89 ++++++- atramhasis/renderers.py | 24 +- atramhasis/scripts/delete_scheme.py | 80 +++--- atramhasis/scripts/initializedb.py | 4 + .../scripts/migrate_sqlalchemyproviders.py | 227 ++++++++++++++++++ atramhasis/scripts/sitemap_generator.py | 18 -- atramhasis/skos/__init__.py | 166 ++----------- atramhasis/utils.py | 40 +++ atramhasis/validators.py | 34 ++- atramhasis/views/crud.py | 50 +++- atramhasis/views/exception_views.py | 68 ++++++ setup.py | 1 + tests/__init__.py | 6 +- tests/processors/__init__.py | 0 tests/processors/test_provider.py | 68 ++++++ tests/scripts/__init__.py | 0 .../test_delete_scheme.py} | 4 +- .../test_migratie_sqlalchemyproviders.py | 70 ++++++ tests/test_datamanagers.py | 35 ++- tests/test_functional.py | 158 +++++++++++- tests/test_mappers.py | 89 +++++++ tests/test_renderes.py | 10 +- tests/test_validation.py | 34 ++- tests/test_views.py | 44 +++- 30 files changed, 1303 insertions(+), 242 deletions(-) create mode 100644 atramhasis/alembic/versions/b2a7d7614973_add_provider_table.py create mode 100644 atramhasis/json_processors/__init__.py create mode 100644 atramhasis/json_processors/provider.py create mode 100644 atramhasis/scripts/migrate_sqlalchemyproviders.py create mode 100644 tests/processors/__init__.py create mode 100644 tests/processors/test_provider.py create mode 100644 tests/scripts/__init__.py rename tests/{test_script_delete_scheme.py => scripts/test_delete_scheme.py} (92%) create mode 100644 tests/scripts/test_migratie_sqlalchemyproviders.py diff --git a/atramhasis/alembic/versions/b2a7d7614973_add_provider_table.py b/atramhasis/alembic/versions/b2a7d7614973_add_provider_table.py new file mode 100644 index 00000000..c42309b2 --- /dev/null +++ b/atramhasis/alembic/versions/b2a7d7614973_add_provider_table.py @@ -0,0 +1,40 @@ +"""Add provider table + +Revision ID: b2a7d7614973 +Revises: 0978347c16f9 +Create Date: 2023-03-22 15:10:27.781804 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "b2a7d7614973" +down_revision = "0978347c16f9" + + +def upgrade(): + op.create_table( + "provider", + sa.Column("id", sa.String(), nullable=False), + sa.Column("conceptscheme_id", sa.Integer(), nullable=False), + sa.Column("uri_pattern", sa.Text(), nullable=False), + sa.Column("metadata", sa.JSON(), nullable=False), + sa.Column( + "expand_strategy", + sa.Enum("RECURSE", "VISIT", name="expandstrategy"), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["conceptscheme_id"], + ["conceptscheme.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("provider") diff --git a/atramhasis/data/datamanagers.py b/atramhasis/data/datamanagers.py index 51c1a1ee..270eadd3 100644 --- a/atramhasis/data/datamanagers.py +++ b/atramhasis/data/datamanagers.py @@ -7,6 +7,7 @@ import uuid from datetime import date from datetime import datetime +from typing import List import dateutil.relativedelta import sqlalchemy as sa @@ -22,12 +23,15 @@ from sqlalchemy import desc from sqlalchemy import func from sqlalchemy import select +from sqlalchemy.orm import Session from sqlalchemy.orm import joinedload from atramhasis.data import popular_concepts from atramhasis.data.models import ConceptVisitLog from atramhasis.data.models import ConceptschemeCounts -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider +from atramhasis.scripts import delete_scheme class DataManager: @@ -35,8 +39,8 @@ class DataManager: A DataManager abstracts all interactions with the database for a certain model. """ - def __init__(self, session): - self.session = session + def __init__(self, session: Session) -> None: + self.session: Session = session class ConceptSchemeManager(DataManager): @@ -383,3 +387,21 @@ def get_most_recent_count_for_scheme(self, conceptscheme_id): .order_by(desc('counted_at')) ).scalar_one() return recent + + +class ProviderDataManager(DataManager): + """A data manager for managing Providers.""" + + def get_provider_by_id(self, provider_id) -> Provider: + return self.session.execute( + select(Provider) + .filter(Provider.id == provider_id) + ).scalar_one() + + def get_all_providers(self) -> List[Provider]: + """ + Retrieve all providers from the database. + + :return: All providers + """ + return self.session.execute(select(Provider)).scalars().all() diff --git a/atramhasis/data/models.py b/atramhasis/data/models.py index 985ca8f5..a00bcf84 100644 --- a/atramhasis/data/models.py +++ b/atramhasis/data/models.py @@ -1,15 +1,33 @@ +import enum + +from skosprovider_sqlalchemy.models import ConceptScheme from sqlalchemy import Column from sqlalchemy import DateTime +from sqlalchemy import Enum +from sqlalchemy import ForeignKey from sqlalchemy import Integer +from sqlalchemy import JSON from sqlalchemy import String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.sql import ( - func -) +from sqlalchemy import Text +from sqlalchemy.orm import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func Base = declarative_base() +class IDGenerationStrategy(enum.Enum): + NUMERIC = enum.auto() + GUID = enum.auto() + MANUAL = enum.auto() + + +class ExpandStrategy(enum.Enum): + RECURSE = 'recurse' + VISIT = 'visit' + + class ConceptschemeVisitLog(Base): __tablename__ = 'conceptscheme_visit_log' id = Column(Integer, primary_key=True, autoincrement=True) @@ -35,3 +53,53 @@ class ConceptschemeCounts(Base): triples = Column(Integer, nullable=False) conceptscheme_triples = Column(Integer, nullable=False) avg_concept_triples = Column(Integer, nullable=False) + + +class Provider(Base): + __tablename__ = 'provider' + + id = Column(String, primary_key=True) + conceptscheme_id = Column( + Integer, + ForeignKey(ConceptScheme.id), + nullable=False, + ) + uri_pattern = Column(Text, nullable=False) + meta = Column('metadata', JSON, nullable=False) # metadata is reserved in sqlalchemy + expand_strategy = Column(Enum(ExpandStrategy)) + + conceptscheme = relationship( + ConceptScheme, uselist=False, single_parent=True, cascade='all, delete-orphan', + ) + + @hybrid_property + def default_language(self): + return self.meta.get('default_language') + + @default_language.setter + def default_language(self, value): + self.meta['default_language'] = value + + @hybrid_property + def force_display_language(self): + return self.meta.get('atramhasis.force_display_language') + + @force_display_language.setter + def force_display_language(self, value): + self.meta['atramhasis.force_display_language'] = value + + @hybrid_property + def id_generation_strategy(self): + return IDGenerationStrategy[self.meta.get('atramhasis.id_generation_strategy')] + + @id_generation_strategy.setter + def id_generation_strategy(self, value): + self.meta['atramhasis.id_generation_strategy'] = value.name + + @hybrid_property + def subject(self): + return self.meta.get('subject') + + @subject.setter + def subject(self, value): + self.meta['subject'] = value diff --git a/atramhasis/json_processors/__init__.py b/atramhasis/json_processors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/atramhasis/json_processors/provider.py b/atramhasis/json_processors/provider.py new file mode 100644 index 00000000..9e1fb35e --- /dev/null +++ b/atramhasis/json_processors/provider.py @@ -0,0 +1,45 @@ +from typing import Mapping + +from skosprovider.registry import Registry +from sqlalchemy.orm import Session + +from atramhasis import mappers +from atramhasis.data.datamanagers import ProviderDataManager +from atramhasis.data.models import Provider +from atramhasis.errors import ValidationError + + +def create_provider(json_data: Mapping, session: Session, skos_registry: Registry) -> Provider: + """Process a provider JSON into a newly stored Provider.""" + for provider in skos_registry.get_providers(): + if provider.get_vocabulary_uri() == json_data["conceptscheme_uri"]: + raise ValidationError( + "Provider could not be validated.", + [{"conceptscheme_uri": "Collides with existing provider."}], + ) + db_provider = mappers.map_provider(json_data) + if not db_provider.id: + # Store conceptscheme first so we can copy its id + session.add(db_provider.conceptscheme) + session.flush() + db_provider.id = str(db_provider.conceptscheme.id) + + session.add(db_provider) + session.flush() + + return db_provider + + +def update_provider(provider_id: str, json_data: Mapping, session: Session) -> Provider: + """Process a JSON into to update an existing provider.""" + manager = ProviderDataManager(session) + db_provider = manager.get_provider_by_id(provider_id) + db_provider = mappers.map_provider(json_data, provider=db_provider) + session.flush() + return db_provider + + +def delete_provider(provider_id, session: Session) -> None: + manager = ProviderDataManager(session) + db_provider = manager.get_provider_by_id(provider_id) + session.delete(db_provider) diff --git a/atramhasis/mappers.py b/atramhasis/mappers.py index f34bece2..077b290b 100644 --- a/atramhasis/mappers.py +++ b/atramhasis/mappers.py @@ -4,12 +4,18 @@ from skosprovider_sqlalchemy.models import Collection from skosprovider_sqlalchemy.models import Concept +from skosprovider_sqlalchemy.models import ConceptScheme from skosprovider_sqlalchemy.models import Label from skosprovider_sqlalchemy.models import Match from skosprovider_sqlalchemy.models import Note from skosprovider_sqlalchemy.models import Source +from skosprovider_sqlalchemy.providers import SQLAlchemyProvider from sqlalchemy.exc import NoResultFound +from atramhasis.data.models import ExpandStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider + def is_html(value): """ @@ -222,3 +228,34 @@ def map_conceptscheme(conceptscheme, conceptscheme_json): source = Source(citation=s.get('citation', '')) conceptscheme.sources.append(source) return conceptscheme + + +def map_provider(provider_json: dict, provider: Provider = None) -> Provider: + """ + Create a atramhasis.data.models.Provider from json data. + + An existing provider can optionally be passed. When passed this one will be updated. + + :param provider_json: JSON data as a dict. + :param provider: A provider which will be updated with the JSON data. When None + a new Provider instance will be returned. + :return: A Provider set with data from the JSON. + """ + if provider is None: + # Only executed on creation. + provider = Provider() + provider.conceptscheme = ConceptScheme(uri=provider_json['conceptscheme_uri']) + provider.id = provider_json.get("id") + + provider.meta = provider_json.get("metadata") or {} + provider.default_language = provider_json.get("default_language") + provider.force_display_language = provider_json.get("force_display_language") + provider.id_generation_strategy = IDGenerationStrategy[ + provider_json.get("id_generation_strategy") or 'NUMERIC' + ] + provider.subject = provider_json.get("subject") or [] + provider.uri_pattern = provider_json["uri_pattern"] + provider.expand_strategy = ExpandStrategy[ + (provider_json.get("expand_strategy") or 'RECURSE').upper() + ] + return provider diff --git a/atramhasis/openapi.yaml b/atramhasis/openapi.yaml index 6d1166bc..16c8be4a 100644 --- a/atramhasis/openapi.yaml +++ b/atramhasis/openapi.yaml @@ -787,6 +787,29 @@ paths: description: Bad request. 5XX: description: Unexpected error. + post: + summary: Create a provider. + description: Create a provider with a conceptscheme. + tags: + - Atramhasis + requestBody: + required: true + description: Data to create provider. + content: + application/json: + schema: + $ref: "#/components/schemas/Provider" + responses: + 201: + description: Provider created successfully + content: + application/json: + schema: + $ref: "#/components/schemas/Provider" + 4XX: + description: Bad request. + 5XX: + description: Unexpected error. /providers/{id}: parameters: - name: id @@ -811,6 +834,45 @@ paths: description: Bad request. 5XX: description: Unexpected error. + put: + summary: Update a provider. + description: Update a provider. + tags: + - Atramhasis + requestBody: + required: true + description: Data to update a provider. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Provider" + - type: object + required: + - id + responses: + 200: + description: Provider updated successfully + content: + application/json: + schema: + $ref: "#/components/schemas/Provider" + 4XX: + description: Bad request. + 5XX: + description: Unexpected error. + delete: + summary: Delete a provider. + description: Delete a provider. + tags: + - Atramhasis + responses: + 204: + description: Provider deleted successfully + 4XX: + description: Bad request. + 5XX: + description: Unexpected error. components: schemas: @@ -1071,39 +1133,56 @@ components: Provider: type: object required: - - id - - type - conceptscheme_uri - uri_pattern - - default_language - - subject - - force_display_language properties: id: type: string + nullable: true + maxLength: 200 type: type: string + nullable: true conceptscheme_uri: type: string + maxLength: 200 + nullable: false uri_pattern: type: string nullable: true + pattern: .*%s.* + maxLength: 200 default_language: type: string nullable: true + maxLength: 100 subject: type: array + nullable: true items: type: string + maxLength: 50 force_display_language: type: string nullable: true + maxLength: 20 id_generation_strategy: type: string + nullable: true enum: - NUMERIC - GUID - MANUAL + expand_strategy: + type: string + nullable: true + enum: + - recurse + - visit + metadata: + type: object + additionalProperties: true + nullable: true Error: type: object required: diff --git a/atramhasis/renderers.py b/atramhasis/renderers.py index 34b6ed36..11cdfc0e 100644 --- a/atramhasis/renderers.py +++ b/atramhasis/renderers.py @@ -1,3 +1,4 @@ +import copy import csv from io import StringIO @@ -22,7 +23,7 @@ from skosprovider_sqlalchemy.models import Source from skosprovider_sqlalchemy.providers import SQLAlchemyProvider -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy class CSVRenderer: @@ -206,24 +207,29 @@ def provider_adapter(provider: VocabularyProvider, *_): else: uri_pattern = None + metadata = copy.deepcopy(provider.metadata) + metadata.pop('id', None) + metadata.pop('conceptscheme_id', None) data = { - 'id': provider.metadata['id'], - 'type': provider.__class__.__name__, - 'conceptscheme_uri': provider.concept_scheme.uri, - 'uri_pattern': uri_pattern, - 'default_language': provider.metadata.get("default_language"), - 'subject': provider.metadata.get("subject"), - 'force_display_language': provider.metadata.get("atramhasis.force_display_language"), + 'id': provider.metadata['id'], + 'type': provider.__class__.__name__, + 'conceptscheme_uri': provider.concept_scheme.uri, + 'uri_pattern': uri_pattern, + 'default_language': metadata.pop("default_language", None), + 'subject': metadata.pop("subject", None), + 'force_display_language': metadata.pop("atramhasis.force_display_language", None), + 'metadata': metadata, } return data def sa_provider_adapter(provider: SQLAlchemyProvider, *args): data = provider_adapter(provider, *args) - strategy = provider.metadata.get( + strategy = data["metadata"].pop( "atramhasis.id_generation_strategy", IDGenerationStrategy.NUMERIC ) data['id_generation_strategy'] = strategy.name + data['expand_strategy'] = provider.expand_strategy return data diff --git a/atramhasis/scripts/delete_scheme.py b/atramhasis/scripts/delete_scheme.py index 6e31ca0e..e9add1ff 100644 --- a/atramhasis/scripts/delete_scheme.py +++ b/atramhasis/scripts/delete_scheme.py @@ -4,72 +4,71 @@ from pyramid.paster import get_appsettings from pyramid.paster import setup_logging -from sqlalchemy import engine_from_config from sqlalchemy.sql.expression import text +from atramhasis import utils + log = logging.getLogger(__name__) -def delete_scheme(settings, scheme_id): - engine = engine_from_config(settings, 'sqlalchemy.') - with engine.begin() as con: - concept_ids = con.execute( - text(f'select id from concept where conceptscheme_id={scheme_id}') - ) - for row in concept_ids: - concept_id = row[0] - delete_concept(concept_id, con) - - con.execute( - text( - f'delete from note where note.id in' - f'(select note_id from conceptscheme_note ' - f'where conceptscheme_id={scheme_id})' - ) +def delete_scheme(session, scheme_id): + concept_ids = session.execute( + text(f'select id from concept where conceptscheme_id={scheme_id}') + ) + for row in concept_ids: + concept_id = row[0] + delete_concept(concept_id, session) + + session.execute( + text( + f'delete from note where note.id in' + f'(select note_id from conceptscheme_note ' + f'where conceptscheme_id={scheme_id})' ) - con.execute( - text( - f'delete from source where source.id in' - f'(select source_id from conceptscheme_source ' - f'where conceptscheme_id={scheme_id})' - ) + ) + session.execute( + text( + f'delete from source where source.id in' + f'(select source_id from conceptscheme_source ' + f'where conceptscheme_id={scheme_id})' ) - con.execute( - text( - f'delete from label where label.id in' - f'(select label_id from conceptscheme_label ' - f'where conceptscheme_id={scheme_id})' - ) + ) + session.execute( + text( + f'delete from label where label.id in' + f'(select label_id from conceptscheme_label ' + f'where conceptscheme_id={scheme_id})' ) - con.execute(text(f'delete from conceptscheme where id = {scheme_id}')) + ) + session.execute(text(f'delete from conceptscheme where id = {scheme_id}')) -def delete_concept(concept_id, con): - con.execute( +def delete_concept(concept_id, session): + session.execute( text( f'delete from note where note.id in' f'(select note_id from concept_note where concept_id={concept_id})' ) ) - con.execute( + session.execute( text( f'delete from source where source.id in' f'(select source_id from concept_source ' f'where concept_id={concept_id})' ) ) - con.execute( + session.execute( text( f'delete from label where label.id in' f'(select label_id from concept_label ' f'where concept_id={concept_id})' ) ) - delete_child_concepts(concept_id, con) - con.execute(text(f'delete from concept where id = {concept_id}')) + delete_child_concepts(concept_id, session) + session.execute(text(f'delete from concept where id = {concept_id}')) -def delete_child_concepts(concept_id, con): +def delete_child_concepts(concept_id, session): select_children = ( text( f'select collection_id_narrower as child ' @@ -81,10 +80,10 @@ def delete_child_concepts(concept_id, con): f'where concept_id_broader = {concept_id}' ) ) - children = con.execute(select_children) + children = session.execute(select_children) for row in children: child_id = row[0] - delete_concept(child_id, con) + delete_concept(child_id, session) def main(): @@ -108,7 +107,8 @@ def main(): if not args.no_input: input("Press [Enter] to continue.") - delete_scheme(settings, args.id) + with utils.db_session(settings) as session: + delete_scheme(session, args.id) if __name__ == '__main__': # pragma: no cover diff --git a/atramhasis/scripts/initializedb.py b/atramhasis/scripts/initializedb.py index aaeb1dda..52845d35 100644 --- a/atramhasis/scripts/initializedb.py +++ b/atramhasis/scripts/initializedb.py @@ -146,3 +146,7 @@ def main(argv=sys.argv): db_session.commit() db_session.close() print('--atramhasis-db-initialized--') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/atramhasis/scripts/migrate_sqlalchemyproviders.py b/atramhasis/scripts/migrate_sqlalchemyproviders.py new file mode 100644 index 00000000..b4a3217c --- /dev/null +++ b/atramhasis/scripts/migrate_sqlalchemyproviders.py @@ -0,0 +1,227 @@ +import argparse +import itertools as it +import json +from datetime import date +from datetime import datetime +from unittest.mock import Mock + +from pyramid.paster import get_appsettings +from pyramid.paster import setup_logging +from pyramid.util import DottedNameResolver +from skosprovider.registry import Registry +from skosprovider.uri import UriPatternGenerator +from skosprovider_sqlalchemy.models import ConceptScheme +from skosprovider_sqlalchemy.providers import SQLAlchemyProvider +from sqlalchemy.orm import Session + +from atramhasis import utils +from atramhasis.data.models import ExpandStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider + + +def get_atramhasis_sqlalchemy_providers(session): + licenses = [ + 'https://creativecommons.org/licenses/by/4.0/', + 'http://data.vlaanderen.be/doc/licentie/modellicentie-gratis-hergebruik/v1.0' + ] + dataseturigenerator = UriPatternGenerator( + 'https://id.erfgoed.net/datasets/thesauri/%s' + ) + return ( + SQLAlchemyProvider( + {'id': 'TREES', 'conceptscheme_id': 1}, + session + ), + SQLAlchemyProvider( + {'id': 'GEOGRAPHY', 'conceptscheme_id': 2}, + session + ), + SQLAlchemyProvider( + { + 'id': 'STYLES', + 'conceptscheme_id': 3, + 'dataset': { + 'uri': dataseturigenerator.generate(id='stijlen_en_culturen'), + 'publisher': ['https://id.erfgoed.net/actoren/501'], + 'created': [date(2008, 2, 14)], + 'language': ['nl-BE'], + 'license': licenses + } + + }, + session, + uri_generator=UriPatternGenerator( + 'https://id.erfgoed.net/thesauri/stijlen_en_culturen/%s' + ) + ), + SQLAlchemyProvider( + { + 'id': 'MATERIALS', + 'conceptscheme_id': 4, + 'dataset': { + 'uri': dataseturigenerator.generate(id='materialen'), + 'publisher': ['https://id.erfgoed.net/actoren/501'], + 'created': [date(2011, 3, 16)], + 'language': ['nl-BE'], + 'license': licenses + } + }, + session, + uri_generator=UriPatternGenerator( + 'https://id.erfgoed.net/thesauri/materialen/%s' + ) + ), + SQLAlchemyProvider( + { + 'id': 'EVENTTYPE', + 'conceptscheme_id': 5, + 'dataset': { + 'uri': dataseturigenerator.generate(id='gebeurtenistypes'), + 'publisher': ['https://id.erfgoed.net/actoren/501'], + 'created': [date(2010, 8, 13)], + 'language': ['nl-BE'], + 'license': licenses + } + }, + session, + uri_generator=UriPatternGenerator( + 'https://id.erfgoed.net/thesauri/gebeurtenistypes/%s' + ) + ), + SQLAlchemyProvider( + { + 'id': 'HERITAGETYPE', + 'conceptscheme_id': 6, + 'dataset': { + 'uri': dataseturigenerator.generate(id='erfgoedtypes'), + 'publisher': ['https://id.erfgoed.net/actoren/501'], + 'created': [date(2008, 2, 14)], + 'language': ['nl-BE'], + 'license': licenses + } + }, + session, + uri_generator=UriPatternGenerator( + 'https://id.erfgoed.net/thesauri/erfgoedtypes/%s' + ) + ), + SQLAlchemyProvider( + { + 'id': 'PERIOD', + 'conceptscheme_id': 7, + 'dataset': { + 'uri': dataseturigenerator.generate(id='dateringen'), + 'publisher': ['https://id.erfgoed.net/actoren/501'], + 'created': [date(2008, 2, 14)], + 'language': ['nl-BE'], + 'license': licenses + } + }, + session, + uri_generator=UriPatternGenerator('https://id.erfgoed.net/thesauri/dateringen/%s') + ), + SQLAlchemyProvider( + { + 'id': 'SPECIES', + 'conceptscheme_id': 8, + 'dataset': { + 'uri': dataseturigenerator.generate(id='soorten'), + 'publisher': ['https://id.erfgoed.net/actoren/501'], + 'created': [date(2011, 5, 23)], + 'language': ['nl-BE', 'la'], + 'license': licenses + }, + 'atramhasis.force_display_label_language': 'la' + }, + session, + uri_generator=UriPatternGenerator('https://id.erfgoed.net/thesauri/soorten/%s') + ), + SQLAlchemyProvider( + { + 'id': 'BLUEBIRDS', + 'conceptscheme_id': 9, + 'atramhasis.id_generation_strategy': IDGenerationStrategy.MANUAL + }, + session, + uri_generator=UriPatternGenerator('https://id.bluebirds.org/%s') + ), + ) + + +def json_serial(obj): + if isinstance(obj, (datetime, date)): + return obj.isoformat() + raise TypeError(f"Type {obj} not serializable") + + +def migrate(skos_registry: Registry, session: Session): + print("Starting migration of SQLAlchemyProvider in the registry...") + for provider in it.chain( + skos_registry.get_providers(), + get_atramhasis_sqlalchemy_providers(session), + ): + if not isinstance(provider, SQLAlchemyProvider): + continue + + if session.get(Provider, provider.conceptscheme_id) is not None: + print( + f"Provider with id {provider.conceptscheme_id} already exists. " + f"Skipping creation of provider for conceptscheme " + f"{provider.metadata.get('id') or ''}" + ) + continue + + print(f" Migrating provider {provider.concept_scheme.uri}") + + if provider.uri_generator: + uri_pattern = getattr(provider.uri_generator, 'pattern', None) + else: + uri_pattern = None + + db_provider = Provider() + if 'atramhasis.id_generation_strategy' in provider.metadata: + # enum must be string to store as json. + provider.metadata['atramhasis.id_generation_strategy'] = ( + provider.metadata['atramhasis.id_generation_strategy'].name + ) + else: + provider.metadata['atramhasis.id_generation_strategy'] = 'NUMERIC' + db_provider.meta = json.loads(json.dumps(provider.metadata, default=json_serial)) + + db_provider.expand_strategy = ExpandStrategy[provider.expand_strategy.upper()] + db_provider.conceptscheme = session.get(ConceptScheme, provider.conceptscheme_id) + db_provider.id = provider.conceptscheme_id + db_provider.uri_pattern = uri_pattern + if 'conceptscheme_id' in db_provider.meta: + del db_provider.meta['conceptscheme_id'] + + session.add(db_provider) + print("Migration finished.") + + +def main(): + parser = argparse.ArgumentParser( + description="Migrate SQLAlchemyProviders from a skosregistry to the database.", + ) + parser.add_argument('settings_file', + help="#") + args = parser.parse_args() + + config_uri = args.settings_file + setup_logging(config_uri) + settings = get_appsettings(config_uri) + + resolver = DottedNameResolver() + skos_registry_factory = resolver.resolve(settings['skosprovider.skosregistry_factory']) + + with utils.db_session(settings) as session: + if settings['skosprovider.skosregistry_location'] == 'registry': + skos_registry = skos_registry_factory() + else: + skos_registry = skos_registry_factory(Mock(db=session)) + migrate(skos_registry, session) + + +if __name__ == '__main__': # pragma: no cover + main() diff --git a/atramhasis/scripts/sitemap_generator.py b/atramhasis/scripts/sitemap_generator.py index 8c952e90..f6e9b197 100644 --- a/atramhasis/scripts/sitemap_generator.py +++ b/atramhasis/scripts/sitemap_generator.py @@ -1,5 +1,4 @@ import argparse -import contextlib import datetime import logging import os @@ -11,29 +10,12 @@ from pyramid.paster import bootstrap from pyramid.paster import get_appsettings from pyramid.paster import setup_logging -from sqlalchemy import engine_from_config -from sqlalchemy.orm import sessionmaker from atramhasis.errors import SkosRegistryNotFoundException log = logging.getLogger(__name__) -@contextlib.contextmanager -def db_session(settings): - engine = engine_from_config(settings, 'sqlalchemy.') - session_maker = sessionmaker(bind=engine) - session = session_maker() - try: - yield session - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - def write_element_to_xml(filename, sitemap_dir, element): tree = ElementTree.ElementTree(element) file_path_name = os.path.join(sitemap_dir, filename) diff --git a/atramhasis/skos/__init__.py b/atramhasis/skos/__init__.py index 719c2867..0584ea26 100644 --- a/atramhasis/skos/__init__.py +++ b/atramhasis/skos/__init__.py @@ -1,15 +1,15 @@ -import enum import logging -from datetime import date import requests from cachecontrol import CacheControl from cachecontrol.heuristics import ExpiresAfter from skosprovider.registry import Registry -from skosprovider.uri import UriPatternGenerator from skosprovider_getty.providers import AATProvider from skosprovider_getty.providers import TGNProvider -from skosprovider_sqlalchemy.providers import SQLAlchemyProvider +from sqlalchemy.orm import Session + +from atramhasis import utils +from atramhasis.data.datamanagers import ProviderDataManager log = logging.getLogger(__name__) LICENSES = [ @@ -18,147 +18,22 @@ ] -class IDGenerationStrategy(enum.Enum): - NUMERIC = enum.auto() - GUID = enum.auto() - MANUAL = enum.auto() +def register_providers_from_db(registry: Registry, session: Session) -> None: + """ + Retrieve all providers stored in the database and add them to the registry. + + :param registry: The registry to which the providers will be added. + :param session: A database session. + """ + manager = ProviderDataManager(session) + for db_provider in manager.get_all_providers(): + provider = utils.db_provider_to_skosprovider(db_provider) + registry.register_provider(provider) def create_registry(request): try: registry = Registry(instance_scope='threaded_thread') - dataseturigenerator = UriPatternGenerator( - 'https://id.erfgoed.net/datasets/thesauri/%s' - ) - - trees = SQLAlchemyProvider( - {'id': 'TREES', 'conceptscheme_id': 1}, - request.db - ) - - geo = SQLAlchemyProvider( - {'id': 'GEOGRAPHY', 'conceptscheme_id': 2}, - request.db - ) - - styles = SQLAlchemyProvider( - { - 'id': 'STYLES', - 'conceptscheme_id': 3, - 'dataset': { - 'uri': dataseturigenerator.generate(id='stijlen_en_culturen'), - 'publisher': ['https://id.erfgoed.net/actoren/501'], - 'created': [date(2008, 2, 14)], - 'language': ['nl-BE'], - 'license': LICENSES - } - - }, - request.db, - uri_generator=UriPatternGenerator( - 'https://id.erfgoed.net/thesauri/stijlen_en_culturen/%s' - ) - ) - - materials = SQLAlchemyProvider( - { - 'id': 'MATERIALS', - 'conceptscheme_id': 4, - 'dataset': { - 'uri': dataseturigenerator.generate(id='materialen'), - 'publisher': ['https://id.erfgoed.net/actoren/501'], - 'created': [date(2011, 3, 16)], - 'language': ['nl-BE'], - 'license': LICENSES - } - }, - request.db, - uri_generator=UriPatternGenerator( - 'https://id.erfgoed.net/thesauri/materialen/%s' - ) - ) - - eventtypes = SQLAlchemyProvider( - { - 'id': 'EVENTTYPE', - 'conceptscheme_id': 5, - 'dataset': { - 'uri': dataseturigenerator.generate(id='gebeurtenistypes'), - 'publisher': ['https://id.erfgoed.net/actoren/501'], - 'created': [date(2010, 8, 13)], - 'language': ['nl-BE'], - 'license': LICENSES - } - }, - request.db, - uri_generator=UriPatternGenerator( - 'https://id.erfgoed.net/thesauri/gebeurtenistypes/%s' - ) - ) - - heritagetypes = SQLAlchemyProvider( - { - 'id': 'HERITAGETYPE', - 'conceptscheme_id': 6, - 'dataset': { - 'uri': dataseturigenerator.generate(id='erfgoedtypes'), - 'publisher': ['https://id.erfgoed.net/actoren/501'], - 'created': [date(2008, 2, 14)], - 'language': ['nl-BE'], - 'license': LICENSES - } - }, - request.db, - uri_generator=UriPatternGenerator( - 'https://id.erfgoed.net/thesauri/erfgoedtypes/%s' - ) - ) - - periods = SQLAlchemyProvider( - { - 'id': 'PERIOD', - 'conceptscheme_id': 7, - 'dataset': { - 'uri': dataseturigenerator.generate(id='dateringen'), - 'publisher': ['https://id.erfgoed.net/actoren/501'], - 'created': [date(2008, 2, 14)], - 'language': ['nl-BE'], - 'license': LICENSES - } - }, - request.db, - uri_generator=UriPatternGenerator('https://id.erfgoed.net/thesauri/dateringen/%s') - ) - - species = SQLAlchemyProvider( - { - 'id': 'SPECIES', - 'conceptscheme_id': 8, - 'dataset': { - 'uri': dataseturigenerator.generate(id='soorten'), - 'publisher': ['https://id.erfgoed.net/actoren/501'], - 'created': [date(2011, 5, 23)], - 'language': ['nl-BE', 'la'], - 'license': LICENSES - }, - 'atramhasis.force_display_label_language': 'la' - }, - request.db, - uri_generator=UriPatternGenerator('https://id.erfgoed.net/thesauri/soorten/%s') - ) - - bluebirds = SQLAlchemyProvider( - { - 'id': 'BLUEBIRDS', - 'conceptscheme_id': 9, - 'atramhasis.id_generation_strategy': IDGenerationStrategy.MANUAL - }, - request.db, - uri_generator=UriPatternGenerator('https://id.bluebirds.org/%s') - ) - - # use 'subject': ['external'] for read only external providers - # (only available in REST service) getty_session = CacheControl(requests.Session(), heuristic=ExpiresAfter(weeks=1)) @@ -172,17 +47,10 @@ def create_registry(request): session=getty_session ) - registry.register_provider(trees) - registry.register_provider(geo) - registry.register_provider(styles) - registry.register_provider(materials) - registry.register_provider(eventtypes) - registry.register_provider(heritagetypes) - registry.register_provider(periods) - registry.register_provider(species) - registry.register_provider(bluebirds) registry.register_provider(aat) registry.register_provider(tgn) + register_providers_from_db(registry, request.db) + return registry except AttributeError: log.exception("Attribute error during creation of Registry.") diff --git a/atramhasis/utils.py b/atramhasis/utils.py index d7f8b66a..59429c30 100644 --- a/atramhasis/utils.py +++ b/atramhasis/utils.py @@ -1,6 +1,8 @@ """ Module containing utility functions used by Atramhasis. """ +import contextlib +import copy from collections import deque from pyramid.httpexceptions import HTTPMethodNotAllowed @@ -10,7 +12,13 @@ from skosprovider.skos import Label from skosprovider.skos import Note from skosprovider.skos import Source +from skosprovider.uri import UriPatternGenerator from skosprovider_sqlalchemy.providers import SQLAlchemyProvider +from sqlalchemy import engine_from_config +from sqlalchemy import orm +from sqlalchemy.orm import sessionmaker + +from atramhasis.data.models import Provider def from_thing(thing): @@ -118,3 +126,35 @@ def label_sort(concepts, language='any'): language=language) ) + +def db_provider_to_skosprovider(db_provider: Provider) -> SQLAlchemyProvider: + """Create a SQLAlchemyProvider from a atramhasis.data.models.Provider. + + :param db_provider: The Provider to use as basis for the SQLAlchemyProvider. + :return: An SQLAlchemyProvider with the data from the `db_provider` + """ + metadata = copy.deepcopy(db_provider.meta) + metadata["conceptscheme_id"] = db_provider.conceptscheme_id + metadata['atramhasis.id_generation_strategy'] = db_provider.id_generation_strategy + metadata["id"] = db_provider.id + return SQLAlchemyProvider( + metadata=metadata, + session=orm.object_session(db_provider), + expand_strategy=db_provider.expand_strategy.value, + uri_generator=UriPatternGenerator(db_provider.uri_pattern), + ) + + +@contextlib.contextmanager +def db_session(settings): + engine = engine_from_config(settings, 'sqlalchemy.') + session_maker = sessionmaker(bind=engine) + session = session_maker() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() diff --git a/atramhasis/validators.py b/atramhasis/validators.py index 247b846f..6a24e229 100644 --- a/atramhasis/validators.py +++ b/atramhasis/validators.py @@ -13,7 +13,7 @@ from sqlalchemy.exc import NoResultFound from atramhasis.errors import ValidationError -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy class ForcedString(colander.String): @@ -690,3 +690,35 @@ def superordinates_hierarchy_rule(errors, node_location, skos_manager, conceptsc 'members', 'collection', 'The superordinates of a collection must not itself be a member of the collection being edited.' ) + + +def validate_provider_json(json_data): + errors = [] + + # Do not allow keys in the metadata which exist in the root of the json. + forbidden_metadata_keys = ( + 'default_language', + 'subject', + 'force_display_language', # while not the same, this would just be confusing + 'atramhasis.force_display_language', + 'id_generation_strategy', # while not the same, this would just be confusing + 'atramhasis.id_generation_strategy', + ) + metadata = json_data.get('metadata', {}) + if wrong_keys := [k for k in forbidden_metadata_keys if k in metadata]: + errors.append({'metadata': f'Found disallowed key(s): {", ".join(wrong_keys)}.'}) + + if json_data.get('default_language'): + if not tags.check(json_data['default_language']): + errors.append({'default_language': 'Invalid language.'}) + + if json_data.get('force_display_language'): + if not tags.check(json_data['force_display_language']): + errors.append({'force_display_language': 'Invalid language.'}) + + if json_data.get('subject'): + if json_data['subject'] != ['hidden']: + errors.append({'subject': 'Subject must be one of: "hidden"'}) + + if errors: + raise ValidationError('Provider could not be validated.', errors) diff --git a/atramhasis/views/crud.py b/atramhasis/views/crud.py index 14d4ae7a..a5ce8006 100644 --- a/atramhasis/views/crud.py +++ b/atramhasis/views/crud.py @@ -7,6 +7,7 @@ import colander import transaction from pyramid.httpexceptions import HTTPMethodNotAllowed +from pyramid.httpexceptions import HTTPNoContent from pyramid.view import view_config from pyramid.view import view_defaults from pyramid_skosprovider.views import ProviderView @@ -16,16 +17,21 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import NoResultFound +from atramhasis import mappers +from atramhasis import utils from atramhasis.audit import audit from atramhasis.cache import invalidate_scheme_cache +from atramhasis.data.datamanagers import ProviderDataManager from atramhasis.errors import ConceptNotFoundException from atramhasis.errors import ConceptSchemeNotFoundException from atramhasis.errors import SkosRegistryNotFoundException from atramhasis.errors import ValidationError +from atramhasis.json_processors import provider from atramhasis.mappers import map_concept from atramhasis.mappers import map_conceptscheme from atramhasis.protected_resources import protected_operation -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.utils import db_provider_to_skosprovider from atramhasis.utils import from_thing from atramhasis.utils import internal_providers_only @@ -281,3 +287,45 @@ def get_providers(self): ) def get_provider(self): return self.request.skos_registry.get_provider(self.request.matchdict["id"]) + + @view_config( + route_name='atramhasis.providers', + permission='add-provider', + request_method='POST', + openapi=True + ) + def add_provider(self): + db_provider = provider.create_provider( + json_data=self.request.openapi_validated.body, + session=self.request.db, + skos_registry=self.request.skos_registry, + ) + self.request.response.status_code = 201 + return utils.db_provider_to_skosprovider(db_provider) + + @view_config( + route_name='atramhasis.provider', + permission='edit-provider', + request_method='PUT', + openapi=True + ) + def update_provider(self): + db_provider = provider.update_provider( + provider_id=self.request.matchdict["id"], + json_data=self.request.openapi_validated.body, + session=self.request.db, + ) + return utils.db_provider_to_skosprovider(db_provider) + + @view_config( + route_name='atramhasis.provider', + permission='delete_provider', + request_method='DELETE', + openapi=True + ) + def delete_provider(self): + provider.delete_provider( + provider_id=self.request.matchdict["id"], + session=self.request.db, + ) + return HTTPNoContent() diff --git a/atramhasis/views/exception_views.py b/atramhasis/views/exception_views.py index d496db97..26196d1c 100644 --- a/atramhasis/views/exception_views.py +++ b/atramhasis/views/exception_views.py @@ -5,9 +5,13 @@ import logging import sys +from openapi_core.unmarshalling.schemas.exceptions import InvalidSchemaValue from pyramid.httpexceptions import HTTPMethodNotAllowed from pyramid.view import notfound_view_config from pyramid.view import view_config +from pyramid_openapi3 import RequestValidationError +from pyramid_openapi3 import ResponseValidationError +from pyramid_openapi3 import openapi_validation_error from skosprovider.exceptions import ProviderUnavailableException from sqlalchemy.exc import IntegrityError @@ -96,3 +100,67 @@ def failed_not_method_not_allowed(exc, request): log.debug(exc.explanation) request.response.status_int = 405 return {'message': exc.explanation} + + +@view_config(context=RequestValidationError, renderer="json") +@view_config(context=ResponseValidationError, renderer="json") +def failed_openapi_validation(exc, request): + try: + errors = [ + _handle_validation_error(validation_error) + for error in exc.errors + if isinstance(error, InvalidSchemaValue) + for validation_error in error.schema_errors + ] + # noinspection PyTypeChecker + errors.extend( + [ + str(error) + for error in exc.errors + if not isinstance(error, InvalidSchemaValue) + ] + ) + request.response.status_int = 400 + if isinstance(exc, RequestValidationError): + subject = "Request" + else: + subject = "Response" + return {"message": f"{subject} was not valid for schema.", "errors": errors} + except Exception: + log.exception("Issue with exception handling.") + return openapi_validation_error(exc, request) + + +def _handle_validation_error(error, path=""): + if error.validator in ("anyOf", "oneOf", "allOf"): + for schema_type in ("anyOf", "oneOf", "allOf"): + if schema_type not in error.schema: + continue + schemas = error.schema.get(schema_type) + break + else: + return None + + response = [] + for i, schema in enumerate(schemas): + schema.pop("x-scope", None) + errors = [ + sub_error + for sub_error in error.context + if sub_error.relative_schema_path[0] == i + ] + if error.path: + schema = {".".join(str(p) for p in error.path): schema} + response.append( + { + "schema": schema, + "errors": [_handle_validation_error(error) for error in errors], + } + ) + return {schema_type: response} + if path: + path += "." + path += ".".join(str(item) for item in error.path) + if not path: + path = "" + return f"{path}: {error.message}" diff --git a/setup.py b/setup.py index b313b932..76402ffd 100644 --- a/setup.py +++ b/setup.py @@ -106,6 +106,7 @@ def run(self): generate_ldf_config = atramhasis.scripts.generate_ldf_config:main sitemap_generator = atramhasis.scripts.sitemap_generator:main delete_scheme = atramhasis.scripts.delete_scheme:main + migrate_sqlalchemy_providers = atramhasis.scripts.migrate_sqlalchemy_providers:main """, cmdclass={ 'prepare': Prepare diff --git a/tests/__init__.py b/tests/__init__.py index c1b5aef0..f49416d6 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -17,9 +17,10 @@ from sqlalchemy.exc import ProgrammingError from sqlalchemy.orm import sessionmaker +from atramhasis import skos from atramhasis.cache import list_region from atramhasis.cache import tree_region -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy from fixtures import data from fixtures import materials as material_data @@ -234,4 +235,7 @@ def create_registry(request): registry.register_provider(test) registry.register_provider(missing_label) registry.register_provider(manual_ids) + + skos.register_providers_from_db(registry, request.db) + return registry diff --git a/tests/processors/__init__.py b/tests/processors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/processors/test_provider.py b/tests/processors/test_provider.py new file mode 100644 index 00000000..13efb4b2 --- /dev/null +++ b/tests/processors/test_provider.py @@ -0,0 +1,68 @@ +from unittest.mock import MagicMock +from unittest.mock import Mock +from unittest.mock import patch + +import pytest +from skosprovider.registry import Registry +from skosprovider_getty.providers import GettyProvider + +from atramhasis.errors import ValidationError +from atramhasis.json_processors import provider + + +def test_create_provider_with_id(): + session = Mock() + with patch.object(provider.mappers, 'map_provider') as mapper: + mapper.return_value = Mock(id='1') + data = {} + response = provider.create_provider(data, session, MagicMock()) + + mapper.assert_called() + session.add.assert_called() + assert response.id != str(response.conceptscheme.id) + + +def test_create_provider_no_id(): + session = Mock() + with patch.object(provider.mappers, 'map_provider') as mapper: + mapper.return_value = Mock(id=None) + data = {} + response = provider.create_provider(data, session, MagicMock()) + + mapper.assert_called() + session.add.assert_called() + assert response.id == str(response.conceptscheme.id) + + +def test_create_provider_duplicate_uri(): + session = Mock() + registry = Registry() + registry.register_provider(GettyProvider({})) + with patch.object(provider.mappers, 'map_provider') as mapper: + mapper.return_value = Mock(id=None) + data = {"conceptscheme_uri": 'http://vocab.getty.edu/aat/'} + with pytest.raises(ValidationError) as e: + provider.create_provider(data, session, registry) + assert e.value.value == 'Provider could not be validated.' + assert e.value.errors == [ + {"conceptscheme_uri": "Collides with existing provider."} + ] + + +def test_update_provider_with_id(): + session = Mock() + with patch.object(provider.mappers, 'map_provider') as mapper: + mapper.return_value = Mock(id='1') + data = {} + provider.update_provider('1', data, session) + + mapper.assert_called() + + +def test_delete_provider_with_id(): + session = Mock() + with patch.object(provider.mappers, 'map_provider') as mapper: + mapper.return_value = Mock(id='1') + provider.delete_provider('1', session) + + session.delete.assert_called() diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_script_delete_scheme.py b/tests/scripts/test_delete_scheme.py similarity index 92% rename from tests/test_script_delete_scheme.py rename to tests/scripts/test_delete_scheme.py index 630a463a..e606c841 100644 --- a/tests/test_script_delete_scheme.py +++ b/tests/scripts/test_delete_scheme.py @@ -20,7 +20,7 @@ from tests import setup_db TEST_DIR = os.path.dirname(__file__) -settings = appconfig('config:' + os.path.join(TEST_DIR, 'conf_test.ini')) +settings = appconfig('config:' + os.path.join(TEST_DIR, '../conf_test.ini')) def setUpModule(): @@ -33,7 +33,7 @@ class DeleteSchemeTest(DbTest): def test_delete(self): with db_session() as session: for scheme_id in range(1, 10): - delete_scheme.delete_scheme(settings, scheme_id) + delete_scheme.delete_scheme(session, scheme_id) assert len(session.execute(select(ConceptScheme)).all()) == 0 assert len(session.execute(select(Concept)).all()) == 0 assert len(session.execute(select(Collection)).all()) == 0 diff --git a/tests/scripts/test_migratie_sqlalchemyproviders.py b/tests/scripts/test_migratie_sqlalchemyproviders.py new file mode 100644 index 00000000..77f5f298 --- /dev/null +++ b/tests/scripts/test_migratie_sqlalchemyproviders.py @@ -0,0 +1,70 @@ +from skosprovider.registry import Registry +from skosprovider_sqlalchemy.models import ConceptScheme +from skosprovider_sqlalchemy.providers import SQLAlchemyProvider +from sqlalchemy import select + +import tests +from atramhasis.data.models import ExpandStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider +from atramhasis.scripts import migrate_sqlalchemyproviders +from tests import DbTest + + +def setUpModule(): + tests.setup_db() + tests.fill_db() + + +class TestMigrateTests(DbTest): + def test_migrate(self): + conceptscheme = ConceptScheme(uri='urn:x-skosprovider:trees') + self.session.add(conceptscheme) + self.session.flush() + + db_providers = self.session.execute(select(Provider)).scalars().all() + self.assertEqual(0, len(db_providers)) + + registry = Registry() + registry.register_provider( + SQLAlchemyProvider( + {"id": "EXTRA", "conceptscheme_id": conceptscheme.id}, self.session + ) + ) + other_providers = migrate_sqlalchemyproviders.get_atramhasis_sqlalchemy_providers( + self.session + ) + for provider in other_providers: + if self.session.get(ConceptScheme, provider.conceptscheme_id) is None: + self.session.add( + ConceptScheme( + id=provider.conceptscheme_id, + uri=f'urn:x-skosprovider:{provider.conceptscheme_id}' + ) + ) + self.session.flush() + + migrate_sqlalchemyproviders.migrate( + skos_registry=registry, + session=self.session, + ) + + db_providers = self.session.execute(select(Provider)).scalars().all() + db_conceptscheme_ids = [p.conceptscheme_id for p in db_providers] + expected_conceptscheme_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, conceptscheme.id] + for expected in expected_conceptscheme_ids: + self.assertIn(expected, db_conceptscheme_ids) + provider = next(p for p in db_providers if p.id == str(conceptscheme.id)) + self.assertEqual(conceptscheme.id, provider.conceptscheme_id) + self.assertEqual(str(conceptscheme.id), provider.id) + self.assertEqual(ExpandStrategy.RECURSE, provider.expand_strategy) + self.assertEqual(IDGenerationStrategy.NUMERIC, provider.id_generation_strategy) + self.assertEqual('urn:x-skosprovider:%s:%s', provider.uri_pattern) + self.assertEqual( + { + 'id': 'EXTRA', + 'subject': [], + 'atramhasis.id_generation_strategy': 'NUMERIC' + }, + provider.meta + ) diff --git a/tests/test_datamanagers.py b/tests/test_datamanagers.py index 9d813a00..b06d57b2 100644 --- a/tests/test_datamanagers.py +++ b/tests/test_datamanagers.py @@ -1,4 +1,3 @@ -import re from datetime import date from datetime import datetime from unittest.mock import Mock @@ -15,10 +14,12 @@ from atramhasis.data.datamanagers import ConceptSchemeManager from atramhasis.data.datamanagers import CountsManager from atramhasis.data.datamanagers import LanguagesManager +from atramhasis.data.datamanagers import ProviderDataManager from atramhasis.data.datamanagers import SkosManager from atramhasis.data.models import ConceptVisitLog from atramhasis.data.models import ConceptschemeCounts -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider from tests import DbTest from tests import fill_db from tests import setup_db @@ -225,3 +226,33 @@ def test_count_for_scheme(self): res = self.counts_manager.get_most_recent_count_for_scheme('TREES') self.assertIsNotNone(res) self.assertEqual(3, res.triples) + + +class ProviderDataManagerTest(DbTest): + def setUp(self): + super().setUp() + self.manager = ProviderDataManager(self.session) + + def test_get_provider_by_id(self): + provider = Provider( + id='a', conceptscheme=ConceptScheme(), uri_pattern='u-p', meta={} + ) + self.session.add(provider) + self.session.flush() + + result = self.manager.get_provider_by_id(provider.id) + self.assertEqual(result, provider) + + def test_get_provider_by_id_no_result(self): + with self.assertRaises(NoResultFound): + self.manager.get_provider_by_id('...') + + def test_get_all_providers(self): + provider = Provider( + id='a', conceptscheme=ConceptScheme(), uri_pattern='u-p', meta={} + ) + self.session.add(provider) + self.session.flush() + + result = self.manager.get_all_providers() + self.assertEqual(result, [provider]) diff --git a/tests/test_functional.py b/tests/test_functional.py index d9d7f1b0..e4c69f83 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -9,12 +9,14 @@ from pyramid.request import Request from skosprovider.exceptions import ProviderUnavailableException from skosprovider.providers import DictionaryProvider +from skosprovider_sqlalchemy.models import ConceptScheme from sqlalchemy.orm import sessionmaker from webtest import TestApp from atramhasis import main from atramhasis.cache import list_region from atramhasis.cache import tree_region +from atramhasis.data.models import Provider from atramhasis.protected_resources import ProtectedResourceEvent from atramhasis.protected_resources import ProtectedResourceException from fixtures.data import chestnut @@ -540,6 +542,155 @@ def test_method_not_allowed(self): def test_get_conceptschemes(self): self.testapp.get('/conceptschemes', headers=self._get_default_headers(), status=200) + def test_create_provider_openapi_validation(self): + response = self.testapp.post_json( + url='/providers', + params={ + 'uri_pattern': 'invalid', + 'subject': 'wrong' + }, + headers=self._get_default_headers(), + expect_errors=True + ) + self.assertEqual( + { + 'message': 'Request was not valid for schema.', + 'errors': [ + ": 'conceptscheme_uri' is a required property", + "uri_pattern: 'invalid' does not match '.*%s.*'", + "subject: 'wrong' is not of type array" + ]}, + response.json + ) + + def test_create_minimal_provider(self): + response = self.testapp.post_json( + url='/providers', + params={ + 'conceptscheme_uri': 'https://id.erfgoed.net/thesauri/conceptschemes', + 'uri_pattern': 'https://id.erfgoed.net/thesauri/erfgoedtypes/%s' + }, + headers=self._get_default_headers(), + status=201 + ) + self.assertEqual( + { + 'id': response.json["id"], + 'type': 'SQLAlchemyProvider', + 'conceptscheme_uri': 'https://id.erfgoed.net/thesauri/conceptschemes', + 'uri_pattern': 'https://id.erfgoed.net/thesauri/erfgoedtypes/%s', + 'default_language': None, + 'subject': [], + 'force_display_language': None, + 'metadata': {}, + 'id_generation_strategy': 'NUMERIC', + 'expand_strategy': 'recurse' + }, + response.json + ) + + def test_create_full_provider(self): + response = self.testapp.post_json( + url='/providers', + params={ + 'id': 'ERFGOEDTYPES', + 'conceptscheme_uri': 'https://id.erfgoed.net/thesauri/conceptschemes', + 'uri_pattern': 'https://id.erfgoed.net/thesauri/erfgoedtypes/%s', + 'default_language': 'NL', + 'force_display_language': 'NL', + 'subject': ['hidden'], + 'metadata': {'Info': 'Extra data about this provider'}, + 'id_generation_strategy': 'MANUAL', + 'expand_strategy': 'visit', + }, + headers=self._get_default_headers(), + status=201 + ) + self.assertEqual( + { + 'id': 'ERFGOEDTYPES', + 'type': 'SQLAlchemyProvider', + 'conceptscheme_uri': 'https://id.erfgoed.net/thesauri/conceptschemes', + 'uri_pattern': 'https://id.erfgoed.net/thesauri/erfgoedtypes/%s', + 'default_language': 'NL', + 'force_display_language': 'NL', + 'subject': ['hidden'], + 'metadata': {'Info': 'Extra data about this provider'}, + 'id_generation_strategy': 'MANUAL', + 'expand_strategy': 'visit', + }, + response.json + ) + + def test_update_provider(self): + conceptscheme = ConceptScheme(uri='https://id.erfgoed.net/thesauri/conceptschemes') + provider = Provider( + id='ERFGOEDTYPES', + uri_pattern='https://id.erfgoed.net/thesauri/erfgoedtypes/%s', + conceptscheme=conceptscheme, + meta={}, + ) + self.session.add(provider) + self.session.flush() + + response = self.testapp.put_json( + url='/providers/ERFGOEDTYPES', + params={ + 'id': 'ERFGOEDTYPES', + 'type': 'SQLAlchemyProvider', + 'conceptscheme_uri': 'https://id.erfgoed.net/thesauri/conceptschemes', + 'uri_pattern': 'https://id.erfgoed.net/thesauri/updated/%s', + 'default_language': 'NL', + 'subject': ['hidden'], + 'force_display_language': 'NL', + 'metadata': {'extra': 'test-extra'}, + 'id_generation_strategy': 'MANUAL', + 'expand_strategy': 'visit' + }, + headers=self._get_default_headers(), + status=200 + ) + + self.assertEqual( + { + 'id': 'ERFGOEDTYPES', + 'type': 'SQLAlchemyProvider', + 'conceptscheme_uri': 'https://id.erfgoed.net/thesauri/conceptschemes', + 'uri_pattern': 'https://id.erfgoed.net/thesauri/updated/%s', + 'default_language': 'NL', + 'subject': ['hidden'], + 'force_display_language': 'NL', + 'metadata': {'extra': 'test-extra'}, + 'id_generation_strategy': 'MANUAL', + 'expand_strategy': 'visit' + }, + response.json + ) + + def test_delete_provider(self): + conceptscheme = ConceptScheme(uri='https://id.erfgoed.net/thesauri/conceptschemes') + provider = Provider( + id='ERFGOEDTYPES', + uri_pattern='https://id.erfgoed.net/thesauri/erfgoedtypes/%s', + conceptscheme=conceptscheme, + meta={}, + ) + self.session.add(provider) + self.session.flush() + conceptscheme_id = conceptscheme.id + + self.session.expire_all() + self.assertIsNotNone(self.session.get(Provider, 'ERFGOEDTYPES')) + + self.testapp.delete( + url='/providers/ERFGOEDTYPES', + headers=self._get_default_headers(), + status=204 + ) + self.session.expire_all() + self.assertIsNone(self.session.get(Provider, 'ERFGOEDTYPES')) + self.assertIsNone(self.session.get(ConceptScheme, conceptscheme_id)) + def test_get_providers(self): response = self.testapp.get( url='/providers', @@ -561,7 +712,8 @@ def test_get_providers(self): 'id': 'TEST', 'subject': ['biology'], 'type': 'DictionaryProvider', - 'uri_pattern': 'urn:x-skosprovider:%s:%s' + 'uri_pattern': 'urn:x-skosprovider:%s:%s', + 'metadata': {}, } ], response.json) @@ -581,7 +733,9 @@ def test_get_provider(self): 'default_language': None, 'subject': [], 'force_display_language': None, - 'id_generation_strategy': 'NUMERIC' + 'id_generation_strategy': 'NUMERIC', + 'metadata': {}, + 'expand_strategy': 'recurse' }, response.json ) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 9e79551f..55cf2cf8 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -10,6 +10,10 @@ from skosprovider_sqlalchemy.models import Source from sqlalchemy.exc import NoResultFound +from atramhasis import mappers +from atramhasis.data.models import ExpandStrategy +from atramhasis.data.models import IDGenerationStrategy +from atramhasis.data.models import Provider from atramhasis.mappers import map_concept from atramhasis.mappers import map_conceptscheme @@ -302,3 +306,88 @@ def test_mapping_html_note(self): self.assertFalse(hasattr(result_concept, 'members')) self.assertEqual('HTML', result_concept.notes[0].markup) self.assertEqual('HTML', result_concept.sources[0].markup) + + +def test_map_provider_new_provider_full(): + data = { + 'metadata': {'meta': 'meta'}, + 'default_language': 'nl', + 'force_display_language': 'nl-force', + 'id_generation_strategy': 'GUID', + 'subject': ['hidden'], + 'uri_pattern': 'uri-pattern', + 'expand_strategy': 'visit', + 'conceptscheme_uri': 'conceptscheme-uri', + 'id': 'p-id' + } + result = mappers.map_provider(data) + assert result.conceptscheme is not None + assert result.conceptscheme.uri == 'conceptscheme-uri' + assert result.id == 'p-id' + assert result.meta == { + 'atramhasis.force_display_language': 'nl-force', + 'atramhasis.id_generation_strategy': 'GUID', + 'default_language': 'nl', + 'meta': 'meta', + 'subject': ['hidden'] + } + assert result.default_language == 'nl' + assert result.force_display_language == 'nl-force' + assert result.id_generation_strategy is IDGenerationStrategy.GUID + assert result.subject == ['hidden'] + assert result.uri_pattern == 'uri-pattern' + assert result.expand_strategy is ExpandStrategy.VISIT + + +def test_map_provider_new_provider_minimal(): + data = { + 'conceptscheme_uri': 'conceptscheme-uri', + 'uri_pattern': 'uri-pattern', + } + result = mappers.map_provider(data) + assert result.conceptscheme is not None + assert result.conceptscheme.uri == 'conceptscheme-uri' + assert result.id is None + assert result.meta == { + 'atramhasis.force_display_language': None, + 'atramhasis.id_generation_strategy': 'NUMERIC', + 'default_language': None, + 'subject': [] + } + assert result.default_language is None + assert result.force_display_language is None + assert result.id_generation_strategy is IDGenerationStrategy.NUMERIC + assert result.subject == [] + assert result.uri_pattern == 'uri-pattern' + assert result.expand_strategy is ExpandStrategy.RECURSE + + +def test_map_provider_existing(): + data = { + 'metadata': {'meta': 'meta'}, + 'default_language': 'nl', + 'force_display_language': 'nl-force', + 'id_generation_strategy': 'GUID', + 'subject': ['hidden'], + 'uri_pattern': 'uri-pattern', + 'expand_strategy': 'visit', + 'id': 'p-id' + } + existing = Provider(id='exists') + result = mappers.map_provider(data, existing) + assert result is existing + assert result.conceptscheme is None + assert result.id == 'exists' + assert result.meta == { + 'atramhasis.force_display_language': 'nl-force', + 'atramhasis.id_generation_strategy': 'GUID', + 'default_language': 'nl', + 'meta': 'meta', + 'subject': ['hidden'] + } + assert result.default_language == 'nl' + assert result.force_display_language == 'nl-force' + assert result.id_generation_strategy is IDGenerationStrategy.GUID + assert result.subject == ['hidden'] + assert result.uri_pattern == 'uri-pattern' + assert result.expand_strategy is ExpandStrategy.VISIT diff --git a/tests/test_renderes.py b/tests/test_renderes.py index bbb16637..34bf5b9c 100644 --- a/tests/test_renderes.py +++ b/tests/test_renderes.py @@ -233,7 +233,8 @@ def test_provider_adapter(self): 'id': 'provider-id', 'subject': 'sub', 'type': 'VocabularyProvider', - 'uri_pattern': 'urn:x-skosprovider:%s:%s' + 'uri_pattern': 'urn:x-skosprovider:%s:%s', + 'metadata': {}, }, result ) @@ -256,7 +257,8 @@ def test_provider_adapter_aat_provider(self): 'id': 'provider-id', 'subject': 'sub', 'type': 'AATProvider', - 'uri_pattern': None + 'uri_pattern': None, + 'metadata': {'uri': 'http://vocab.getty.edu/aat/'}, }, result ) @@ -289,10 +291,12 @@ def test_provider_adapter_sqlalchemy_provider(self): 'default_language': 'nl-be', 'force_display_language': 'force-nl', 'id': 'provider-id', + 'expand_strategy': 'recurse', 'id_generation_strategy': 'NUMERIC', 'subject': 'sub', 'type': 'SQLAlchemyProvider', - 'uri_pattern': 'urn:x-skosprovider:%s:%s' + 'uri_pattern': 'urn:x-skosprovider:%s:%s', + 'metadata': {}, }, result ) diff --git a/tests/test_validation.py b/tests/test_validation.py index 65fbef69..944da74f 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -8,8 +8,9 @@ from skosprovider_sqlalchemy.models import LabelType from sqlalchemy.exc import NoResultFound +from atramhasis import validators from atramhasis.errors import ValidationError -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy from atramhasis.validators import Concept as ConceptSchema from atramhasis.validators import ConceptScheme as ConceptSchemeSchema from atramhasis.validators import LanguageTag @@ -813,3 +814,34 @@ def test_id_generation_manual_not_unique(self): validate_id_generation=False, ) concept_schema.deserialize(self.json_concept) + + def test_validate_provider(self): + """ + This looks like it does not validate much, but openapi validates the majority. + + We only need to manually validate what openapi cannot. + """ + json_data = {'metadata': {}} + validators.validate_provider_json(json_data) + + def test_validate_provider_forbidden_keys(self): + forbidden_metadata_keys = ( + 'default_language', + 'subject', + 'force_display_language', + 'atramhasis.force_display_language', + 'id_generation_strategy', + 'atramhasis.id_generation_strategy', + ) + for bad_key in forbidden_metadata_keys: + json_data = {'metadata': {bad_key: 'value'}} + with self.assertRaises(ValidationError): + validators.validate_provider_json(json_data) + + def test_validate_provider_bad_language(self): + json_data = {'default_language': 'notalanguage'} + with self.assertRaises(ValidationError): + validators.validate_provider_json(json_data) + json_data = {'force_display_language': 'notalanguage'} + with self.assertRaises(ValidationError): + validators.validate_provider_json(json_data) diff --git a/tests/test_views.py b/tests/test_views.py index eae4684a..6a250dd9 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -2,10 +2,13 @@ import unittest from unittest.mock import Mock +import mock import pytest +from openapi_core.validation.request.datatypes import RequestValidationResult from paste.deploy.loadwsgi import appconfig from pyramid import testing from pyramid.config.settings import Settings +from pyramid.httpexceptions import HTTPNoContent from pyramid.request import apply_request_extensions from pyramid.testing import DummyRequest from skosprovider.registry import Registry @@ -19,6 +22,7 @@ from skosprovider_sqlalchemy.models import Thing from skosprovider_sqlalchemy.providers import SQLAlchemyProvider from sqlalchemy.exc import NoResultFound +from sqlalchemy.orm import Session from webob.multidict import MultiDict from skosprovider.skos import Concept as SkosConcept @@ -27,7 +31,7 @@ from atramhasis.errors import ConceptSchemeNotFoundException from atramhasis.errors import SkosRegistryNotFoundException from atramhasis.errors import ValidationError -from atramhasis.skos import IDGenerationStrategy +from atramhasis.data.models import IDGenerationStrategy from atramhasis.views.crud import AtramhasisCrud from atramhasis.views.views import AtramhasisAdminView from atramhasis.views.views import AtramhasisListView @@ -724,3 +728,41 @@ def test_add_concept_manual_id_strategy(self): db_concept.conceptscheme = ConceptScheme(id=1, uri='urn:x-skosprovider:trees') self.request.data_managers["skos_manager"].get_thing = lambda *_: db_concept self.view.edit_concept() + + def test_add_provider(self): + self.request.openapi_validated = RequestValidationResult(body={}) + self.request.skos_registry = Registry() + view = 'atramhasis.views.crud' + + with mock.patch(f'{view}.provider.create_provider', autospec=True) as processor, \ + mock.patch(f'{view}.utils.db_provider_to_skosprovider', + autospec=True) as renderer: + + response = self.view.add_provider() + self.assertEqual(201, self.request.response.status_code) + processor.assert_called() + renderer.assert_called() + self.assertEqual(response, renderer.return_value) + + def test_update_provider(self): + self.request.openapi_validated = RequestValidationResult(body={}) + self.request.matchdict = {"id": 1} + view = 'atramhasis.views.crud' + + with mock.patch(f'{view}.provider.update_provider', autospec=True) as processor, \ + mock.patch(f'{view}.utils.db_provider_to_skosprovider', + autospec=True) as renderer: + + response = self.view.update_provider() + processor.assert_called() + renderer.assert_called() + self.assertEqual(response, renderer.return_value) + + def test_delete_provider(self): + self.request.matchdict = {"id": 1} + view = 'atramhasis.views.crud' + + with mock.patch(f'{view}.provider.delete_provider', autospec=True) as processor: + response = self.view.delete_provider() + processor.assert_called() + self.assertIsInstance(response, HTTPNoContent)