Skip to content

Commit

Permalink
Make cs optional for import. Refs #100
Browse files Browse the repository at this point in the history
  • Loading branch information
@koenedaele
koenedaele committed Mar 29, 2023
1 parent ccfa585 commit 2a0b7cf
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 22 deletions.
24 changes: 22 additions & 2 deletions skosprovider_sqlalchemy/utils.py
View file
Expand Up @@ -3,9 +3,12 @@
from language_tags import tags
from skosprovider.skos import Collection
from skosprovider.skos import Concept
from skosprovider.providers import VocabularyProvider
from sqlalchemy import select
from sqlalchemy.exc import NoResultFound
from sqlalchemy.orm.session import Session

from skosprovider_sqlalchemy.models import ConceptScheme as ConceptSchemeModel
from skosprovider_sqlalchemy.models import Collection as CollectionModel
from skosprovider_sqlalchemy.models import Concept as ConceptModel
from skosprovider_sqlalchemy.models import Label as LabelModel
Expand All @@ -18,7 +21,7 @@
log = logging.getLogger(__name__)


def import_provider(provider, conceptscheme, session):
def import_provider(provider: VocabularyProvider, session: Session, conceptscheme: ConceptSchemeModel = None) -> ConceptSchemeModel:
'''
Import a provider into a SQLAlchemy database.
Expand All @@ -28,13 +31,27 @@ def import_provider(provider, conceptscheme, session):
:param conceptscheme: A
:class:`skosprovider_sqlalchemy.models.Conceptscheme` to import
the provider into. This should be an empty scheme so that there are
no possible id clashes.
no possible id clashes. If no conceptscheme is provided, one will be
created.
:param session: A :class:`sqlalchemy.orm.session.Session`.
:return: The conceptscheme that holds the concepts and collections. Either
the same conceptscheme that was passed into the provider, or the one that
was created by this function.
:rtype: skosprovider_sqlalchemy.models.Conceptscheme
'''

if not conceptscheme:
conceptscheme = ConceptSchemeModel()
session.add(conceptscheme)

# Copy information about the scheme
cs = provider.concept_scheme
if not conceptscheme.uri:
conceptscheme.uri = cs.uri
elif cs.uri and conceptscheme.uri != cs.uri:
log.warning('Conceptscheme model has different URI than conceptscheme attached to provider.')

_add_labels(conceptscheme, cs.labels, session)
_add_notes(conceptscheme, cs.notes, session)
_add_sources(conceptscheme, cs.sources, session)
Expand All @@ -45,6 +62,7 @@ def import_provider(provider, conceptscheme, session):
# First pass: load all concepts and collections
for stuff in provider.get_all():
c = provider.get_by_id(stuff['id'])
log.warning(c)
if isinstance(c, Concept):
cm = ConceptModel(
concept_id=c.id,
Expand Down Expand Up @@ -73,6 +91,7 @@ def import_provider(provider, conceptscheme, session):
# Second pass: link
for stuff in provider.get_all():
c = provider.get_by_id(stuff['id'])
log.warning(c)
if isinstance(c, Concept):
cm = session.execute(
select(ConceptModel)
Expand Down Expand Up @@ -148,6 +167,7 @@ def import_provider(provider, conceptscheme, session):
log.warning(
'Tried to add a relation %s member %s, but target \
does not exist. Relation will be lost.' % (c.id, mc))
return conceptscheme


def _check_language(language_tag, session):
Expand Down
81 changes: 61 additions & 20 deletions tests/test_utils.py
View file
Expand Up @@ -8,6 +8,7 @@
from skosprovider_sqlalchemy.models import Initialiser
from skosprovider_sqlalchemy.utils import VisitationCalculator
from skosprovider_sqlalchemy.utils import import_provider

from tests import DBTestCase


Expand Down Expand Up @@ -139,6 +140,8 @@ def _get_geo():

def _get_buildings():
from skosprovider.providers import DictionaryProvider
from skosprovider.uri import UriPatternGenerator
from skosprovider.skos import ConceptScheme

buildings = DictionaryProvider(
{'id': 'BUILDINGS'},
Expand All @@ -157,7 +160,7 @@ def _get_buildings():
'exact': ['http://vocab.getty.edu/aat/300006888']
}
}, {
'id': 2,
'id': '2',
'labels': [
{'type': 'prefLabel', 'language': 'en', 'label': 'Castle'}
],
Expand All @@ -166,7 +169,7 @@ def _get_buildings():
'broad': ['http://vocab.getty.edu/aat/300006888']
}
}, {
'id': 3,
'id': '3',
'labels': [
{
'type': 'prefLabel',
Expand All @@ -179,7 +182,7 @@ def _get_buildings():
'close': ['http://vocab.getty.edu/aat/300005425']
}
}, {
'id': 4,
'id': '4',
'labels': [
{'type': 'prefLabel', 'language': 'en', 'label': 'Huts'},
{'type': 'prefLabel', 'language': None, 'label': 'Hutten'}
Expand All @@ -189,13 +192,19 @@ def _get_buildings():
'exact': ['http://vocab.getty.edu/aat/300004824']
}
}
]
],
concept_scheme = ConceptScheme(
uri = 'https://id.buildings.org'
),
uri_generator = UriPatternGenerator('https://id.buildings.org/%s')
)
return buildings


def _get_materials():
from skosprovider.providers import DictionaryProvider
from skosprovider.uri import UriPatternGenerator
from skosprovider.skos import ConceptScheme

materials = DictionaryProvider(
{'id': 'MATERIALS'},
Expand Down Expand Up @@ -224,7 +233,11 @@ def _get_materials():
],
'members': [654]
}
]
],
concept_scheme = ConceptScheme(
uri = 'https://id.materials.org'
),
uri_generator = UriPatternGenerator('https://id.materials.org/%s')
)
return materials

Expand Down Expand Up @@ -312,7 +325,7 @@ def test_empty_provider(self):
p = DictionaryProvider({'id': 'EMPTY'}, [])
cs = self._get_cs()
self.session.add(cs)
import_provider(p, cs, self.session)
import_provider(p, self.session, cs)
scheme = self.session.get(ConceptSchemeModel, 68)
assert scheme == cs

Expand All @@ -327,7 +340,7 @@ def test_string_concept_id_provider(self):
)
cs = self._get_cs()
self.session.add(cs)
import_provider(p, cs, self.session)
import_provider(p, self.session, cs)
scheme = self.session.get(ConceptSchemeModel, 68)
assert scheme == cs
assert len(scheme.concepts) == 1
Expand All @@ -341,7 +354,7 @@ def test_menu(self):
csvprovider = _get_menu()
cs = self._get_cs()
self.session.add(cs)
import_provider(csvprovider, cs, self.session)
import_provider(csvprovider, self.session, cs)
lobster = self.session.execute(
select(ConceptModel)
.filter(
Expand All @@ -363,7 +376,7 @@ def test_geo(self):
geoprovider = _get_geo()
cs = self._get_cs()
self.session.add(cs)
import_provider(geoprovider, cs, self.session)
import_provider(geoprovider, self.session, cs)
self.session.flush()
self.session.expire_all()
world = self.session.execute(
Expand Down Expand Up @@ -413,7 +426,7 @@ def test_buildings(self):
buildingprovider = _get_buildings()
cs = self._get_cs()
self.session.add(cs)
import_provider(buildingprovider, cs, self.session)
import_provider(buildingprovider, self.session, cs)
castle = self.session.execute(
select(ConceptModel)
.filter(
Expand All @@ -434,6 +447,24 @@ def test_buildings(self):
assert 'exactMatch' == hut.matches[0].matchtype_id
assert 'http://vocab.getty.edu/aat/300004824' == hut.matches[0].uri

def test_conceptscheme_is_optional(self):
from skosprovider_sqlalchemy.models import (
Concept as ConceptModel
)

buildingprovider = _get_buildings()
cs = import_provider(buildingprovider, self.session)
assert 'https://id.buildings.org' == cs.uri

castle = self.session.execute(
select(ConceptModel)
.filter(
ConceptModel.conceptscheme == cs,
ConceptModel.concept_id == '2'
)
).scalar_one()
assert 2 == len(castle.broader_concepts)

def test_heritage_types(self):
from skosprovider_sqlalchemy.models import (
Concept as ConceptModel,
Expand All @@ -442,7 +473,7 @@ def test_heritage_types(self):
heritagetypesprovider = _get_heritage_types()
cs = self._get_cs()
self.session.add(cs)
import_provider(heritagetypesprovider, cs, self.session)
import_provider(heritagetypesprovider, self.session, cs)
bomen = self.session.execute(
select(ConceptModel)
.filter(
Expand All @@ -464,7 +495,7 @@ def test_event_types(self):
eventtypesprovider = _get_event_types()
cs = self._get_cs()
self.session.add(cs)
import_provider(eventtypesprovider, cs, self.session)
import_provider(eventtypesprovider, self.session, cs)
archeologische_opgravingen = self.session.execute(
select(ConceptModel)
.filter(
Expand All @@ -482,13 +513,23 @@ def test_materials(self):
materialsprovider = _get_materials()
cs = self._get_cs()
self.session.add(cs)
import_provider(materialsprovider, cs, self.session)
import_provider(materialsprovider, self.session, cs)
materials = self.session.execute(
select(ThingModel)
.filter(ThingModel.conceptscheme == cs)
).scalars().all()
assert 2 == len(materials)

def test_materials_cs_is_created(self):
materialsprovider = _get_materials()
cs = self._get_cs()
self.session.add(cs)
cs = import_provider(materialsprovider, self.session, cs)

materialsprovider = _get_materials()
cs_from_provider = import_provider(materialsprovider, self.session)

assert cs_from_provider.id != cs.id

class TestVisitationCalculator(DBTestCase):

Expand Down Expand Up @@ -518,7 +559,7 @@ def test_empty_provider(self):
p = DictionaryProvider({'id': 'EMPTY'}, [])
cs = self._get_cs()
self.session.add(cs)
import_provider(p, cs, self.session)
import_provider(p, self.session, cs)
vc = VisitationCalculator(self.session)
v = vc.visit(cs)
assert 0 == len(v)
Expand All @@ -541,13 +582,13 @@ def test_provider_invalid_language(self):
])
cs = self._get_cs()
self.session.add(cs)
import_provider(p, cs, self.session)
import_provider(p, self.session, cs)

def test_menu(self):
csvprovider = _get_menu()
cs = self._get_cs()
self.session.add(cs)
import_provider(csvprovider, cs, self.session)
import_provider(csvprovider, self.session, cs)
vc = VisitationCalculator(self.session)
visit = vc.visit(cs)
assert 11 == len(visit)
Expand All @@ -559,7 +600,7 @@ def test_menu_sorted(self):
csvprovider = _get_menu()
cs = self._get_cs()
self.session.add(cs)
import_provider(csvprovider, cs, self.session)
import_provider(csvprovider, self.session, cs)
vc = VisitationCalculator(self.session)
visit = vc.visit(cs)
assert 11 == len(visit)
Expand All @@ -576,7 +617,7 @@ def test_geo(self):
geoprovider = _get_geo()
cs = self._get_cs()
self.session.add(cs)
import_provider(geoprovider, cs, self.session)
import_provider(geoprovider, self.session, cs)
vc = VisitationCalculator(self.session)
visit = vc.visit(cs)
assert 10 == len(visit)
Expand All @@ -601,13 +642,13 @@ def test_buildings(self):
buildingprovider = _get_buildings()
cs = self._get_cs()
self.session.add(cs)
import_provider(buildingprovider, cs, self.session)
import_provider(buildingprovider, self.session, cs)
vc = VisitationCalculator(self.session)
visit = vc.visit(cs)
assert len(visit) == 5
# Check that castle is present twice
ids = [self.session.get(ConceptModel, v['id']).concept_id for v in visit]
assert ids.count(2) == 2
assert ids.count('2') == 2
for v in visit:
# Check that fortification has one child
if v['id'] == 1:
Expand Down

0 comments on commit 2a0b7cf

Please sign in to comment.