Merge pull request #75 from OnroerendErfgoed/DEV_0.5.0

Dev 0.5.0

.travis.yml

@@ -2,10 +2,9 @@ sudo: false
 language: python
 python:
 - 2.7
-- 3.3
-- 3.4
-- 3.5
 - 3.6
+- 3.7
+- 3.8
 install:
 - pip install -U setuptools
 - pip install -r requirements.txt #fix versions

CHANGES.rst

@@ -1,3 +1,11 @@
+0.5.0 (2020-08-06)
+------------------
+
+- Compatibile with `SkosProvider 0.7.0 <http://skosprovider.readthedocs.io/en/0.7.0/>`_. (#59)
+- Prevent unnecessary loading of conceptschemes. (#56)
+- Update to RDFlib 5.0.0 (#69)
+- Supports Python 2.7, 3.6, 3.7 and 3.8. Last version to support Python 2.
+
 0.4.2 (2017-09-06)
 ------------------
 
@@ -12,7 +20,7 @@
 ------------------
 
 - Stop collecting SKOS Concept and Collection subclasses. They are now included
-  in the code base since they seem to have become rather stable and the reduces
+  in the code base since they seem to have become rather stable and this reduces
   the startup time of the provider significantly. (#28)
 - Add support for python 3.6 when testing.
 

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2016 Onroerend Erfgoed
+Copyright (c) 2014-2020 Onroerend Erfgoed
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

docs/source/conf.py

@@ -50,16 +50,16 @@
 
 # General information about the project.
 project = u'skosprovider_getty'
-copyright = u'2014-2017, Flanders Heritage Agency'
+copyright = u'2014-2020, Flanders Heritage Agency'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.4'
+version = '0.5'
 # The full version, including alpha/beta/rc tags.
-release = '0.4.2'
+release = '0.5.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

requirements-dev.txt

@@ -1,9 +1,12 @@
 # Runtime requirements
 --requirement requirements.txt
 
-# Documentation
-Sphinx==1.7.5
+# Testing
+pytest==4.6.9 ; python_version < '3.0'
+pytest==5.3.5 ; python_version >= '3.0'
+pytest-cov==2.8.1
+coveralls==1.11.1
 
-#testing
-pytest==3.6.0
-pytest-cov==2.5.1
+# Documentation
+Sphinx==1.8.5 ; python_version < '3.5'
+Sphinx==3.0.4 ; python_version >= '3.5'

requirements.txt

@@ -1,4 +1,4 @@
-requests==2.18.4
-skosprovider==0.6.1
-#-e git+https://github.com/koenedaele/skosprovider.git#egg=skosprovider
-rdflib==4.2.2
+requests==2.23.0
+skosprovider==0.7.1
+#-e git+https://github.com/koenedaele/skosprovider.git@DEV_0.7.0#egg=skosprovider
+rdflib==5.0.0

setup.py

@@ -14,16 +14,17 @@
 ]
 
 requires = [
-    'skosprovider>=0.6.0',
+    'skosprovider>=0.7.0',
     'requests',
     'rdflib'
 ]
 
 setup(
     name='skosprovider_getty',
-    version='0.4.2',
+    version='0.5.0',
     description='Skosprovider implementation of the Getty Vocabularies',
     long_description=README + '\n\n' + CHANGES,
+    long_description_content_type='text/x-rst',
     packages=packages,
     include_package_data=True,
     install_requires=requires,
@@ -35,14 +36,13 @@
         'License :: OSI Approved :: MIT License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6'
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
     ],
     author='Flanders Heritage Agency',
     author_email='ict@onroerenderfgoed.be',
     url='https://github.com/OnroerendErfgoed/skosprovider_getty',
-    keywords='getty skos skosprovider vocabulary AAT TGNi ULAN',
+    keywords='getty skos skosprovider vocabulary AAT TGN ULAN',
     test_suite='nose.collector'
 )

skosprovider_getty/providers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 '''
-This module contains classes that implement 
+This module contains classes that implement
 :class:`skosprovider.providers.VocabularyProvider` against the LOD version of
 the Getty Vocabularies (AAT, TGN and ULAN).
 
@@ -15,15 +15,16 @@
 import logging
 
 from language_tags import tags
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, Timeout
 from skosprovider.exceptions import ProviderUnavailableException
 from skosprovider.providers import VocabularyProvider
 from skosprovider_getty.utils import (
     uri_to_id, uri_to_graph,
     conceptscheme_from_uri,
     things_from_graph,
     SubClassCollector,
-    GVP
+    GVP,
+    do_get_request
 )
 
 log = logging.getLogger(__name__)
@@ -52,20 +53,29 @@ def __init__(self, metadata, **kwargs):
             metadata['default_language'] = 'en'
         if 'subject' not in metadata:
             metadata['subject'] = []
-        self.metadata = metadata
         self.base_url = kwargs.get('base_url', 'http://vocab.getty.edu/')
         self.vocab_id = kwargs.get('vocab_id', 'aat')
         self.url = kwargs.get('url', self.base_url + self.vocab_id)
+        if 'uri' not in metadata:
+            metadata['uri'] = self.url + '/'
+        self.metadata = metadata
         self.subclasses = kwargs.get('subclasses', SubClassCollector(GVP))
         self.session = kwargs.get('session', requests.Session())
+        self.allowed_instance_scopes = kwargs.get(
+            'allowed_instance_scopes',
+            ['single', 'threaded_thread']
+        )
+        self._conceptscheme = None
 
     @property
     def concept_scheme(self):
-        return self._get_concept_scheme()
+        if self._conceptscheme is None:
+            self._conceptscheme = self._get_concept_scheme()
+        return self._conceptscheme
 
     def _get_concept_scheme(self):
         return conceptscheme_from_uri(
-            self.url,
+            self.metadata['uri'],
             session=self.session
         )
 
@@ -86,7 +96,12 @@ def get_by_id(self, id, change_notes=False):
             log.debug('Failed to retrieve data for %s/%s.rdf' % (self.url, id))
             return False
         # get the concept
-        things = things_from_graph(graph, self.subclasses, self.concept_scheme)
+        things = things_from_graph(
+            graph,
+            self.subclasses,
+            self.concept_scheme,
+            session=self.session
+        )
         if len(things) == 0:
             return False
         c = things[0]
@@ -186,6 +201,18 @@ def find(self, query, **kwargs):
             if coll_depth not in ('members', 'all'):
                 raise ValueError(
                     "collection - 'depth': only the following values are allowed: 'members', 'all'")
+        #Matches (optional)
+        match_uri = None
+        match_pred = 'skos:mappingRelation'
+        if 'matches' in query:
+            match_uri = query['matches'].get('uri', None)
+            if not match_uri:
+                raise ValueError(
+                    'Please provide a URI to match with.'
+                )
+            match_type = query['matches'].get('type', None)
+            if match_type:
+                match_pred = 'skos:%sMatch' % match_type
 
         #build sparql query
         coll_x = ""
@@ -194,6 +221,9 @@ def find(self, query, **kwargs):
         elif coll_id is not None and coll_depth == 'members':
             coll_x = "gvp:broader " + self.vocab_id + ":" + coll_id + ";"
 
+        match_values = ""
+        if match_uri is not None:
+            match_values = "%s <%s>;" % (match_pred, match_uri)
 
         type_values = "((?Type = skos:Concept) || (?Type = skos:Collection))"
         if type_c == 'concept':
@@ -202,12 +232,15 @@ def find(self, query, **kwargs):
             type_values = "(?Type = skos:Collection)"
         query = """
             SELECT ?Subject ?Term ?Type ?Id (lang(?Term) as ?Lang) {
-            ?Subject rdf:type ?Type; dc:identifier ?Id; %s  skos:inScheme %s:; %s.
+            ?Subject rdf:type ?Type; dc:identifier ?Id; skos:inScheme %s:; %s%s%s.
                             OPTIONAL {
                   {?Subject xl:prefLabel [skosxl:literalForm ?Term]}
                           }
             FILTER(%s)
-            }""" % (self._build_keywords(label), self.vocab_id, coll_x, type_values)
+            }""" % (
+                self.vocab_id,
+                self._build_keywords(label), coll_x, match_values,
+                type_values)
         ret= self._get_answer(query, **kwargs)
         language = self._get_language(**kwargs)
         sort = self._get_sort(**kwargs)
@@ -238,14 +271,7 @@ def _get_answer(self, query, **kwargs):
             * label: A label to represent the concept or collection.
         """
         request = self.base_url + "sparql.json"
-        try:
-            res = self.session.get(request, params={"query": query})
-        except ConnectionError as e:
-            raise ProviderUnavailableException("Request could not be executed - Request: %s - Params: %s" % (request, query))
-        if res.status_code == 404:
-            raise ProviderUnavailableException("Service not found (status_code 404) - Request: %s - Params: %s" % (request, query))
-        if not res.encoding:
-            res.encoding = 'utf-8'
+        res = do_get_request(request, self.session, params={'query': query})
         r = res.json()
         d = {}
         for result in r["results"]["bindings"]:
@@ -257,7 +283,7 @@ def _get_answer(self, query, **kwargs):
             item = {
             'id': result["Id"]["value"],
             'uri': uri,
-            'type': result["Type"]["value"].rsplit('#', 1)[1],
+            'type': result["Type"]["value"].rsplit('#', 1)[1].lower(),
             'label': label,
             'lang': result["Lang"]["value"]
             }
@@ -365,8 +391,8 @@ def expand(self, id):
                 """ % (self.vocab_id, self.vocab_id + ":" + id, id, self.vocab_id)
 
         print (query)
-        res = self.session.get(self.base_url + "sparql.json", params={"query": query})
-        res.encoding = 'utf-8'
+        request = self.base_url + "sparql.json"
+        res = do_get_request(request, self.session, params={'query': query})
         r = res.json()
 
         result = [result['Id']['value'] for result in r['results']['bindings']]