Move to pypi RDFlib-SQLAlchemy dependency (#64)

* Move to pypi RDFlib-SQLAlchemy dependency

* Register rdflib_sqlalchemy plugin manually

* Make rdflib-sqlalchemy store from PyPI work

* Add test for exception when adding fact to graph

gutenberg/acquire/metadata.py

@@ -1,7 +1,7 @@
 """Module to deal with metadata acquisition."""
 # pylint:disable=W0603
 
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 import abc
 import logging
@@ -17,6 +17,8 @@
 from rdflib.graph import Graph
 from rdflib.store import Store
 from rdflib.term import URIRef
+from rdflib_sqlalchemy import registerplugins
+from six import text_type
 from six import with_metaclass
 
 from gutenberg._domain_model.exceptions import CacheAlreadyExistsException
@@ -91,7 +93,13 @@ def populate(self):
         with closing(self.graph):
             with self._download_metadata_archive() as metadata_archive:
                 for fact in self._iter_metadata_triples(metadata_archive):
-                    self.graph.add(fact)
+                    self._add_to_graph(fact)
+
+    def _add_to_graph(self, fact):
+        """Adds a (subject, predicate, object) RDF triple to the graph.
+
+        """
+        self.graph.add(fact)
 
     def _populate_setup(self):
         """Executes operations necessary before the cache can be populated.
@@ -208,9 +216,31 @@ def __init__(self, cache_location):
     def _local_storage_path(self):
         return self.cache_uri[len(self._CACHE_URI_PREFIX):]
 
+    def _add_to_graph(self, fact):
+        try:
+            self.graph.add(fact)
+        except Exception as ex:
+            self.graph.rollback()
+            if not self._is_graph_add_exception_acceptable(ex):
+                raise ex
+        else:
+            self.graph.commit()
+
+    @classmethod
+    def _is_graph_add_exception_acceptable(cls, ex):
+        """Checks if a graph-add exception can be safely ignored.
+
+        """
+        # integrity errors due to violating unique constraints should be safe to
+        # ignore since the only unique constraints in rdflib-sqlalchemy are on
+        # index columns
+        return 'UNIQUE constraint failed' in text_type(ex)
+
 
 _METADATA_CACHE = None
 
+registerplugins()
+
 
 def set_metadata_cache(cache):
     """Sets the metadata cache object to use.

requirements.pip

@@ -3,4 +3,4 @@ rdflib>=4.2.0
 requests>=2.5.1
 six>=1.10.0
 setuptools>=18.5
-git+https://github.com/RDFLib/rdflib-sqlalchemy.git@20fbf1fbbdbc53917bfd7554f38d22d08e71e225#egg=rdflib-sqlalchemy-0.2.dev0
+rdflib-sqlalchemy>=0.3.8

tests/test_metadata_cache.py

@@ -15,6 +15,7 @@
 from gutenberg.acquire.metadata import SqliteMetadataCache
 from gutenberg.acquire.metadata import set_metadata_cache
 from gutenberg.query import get_metadata
+from tests._util import always_throw
 from tests._util import unittest
 
 
@@ -105,6 +106,15 @@ def setUp(self):
         self.cache = SqliteMetadataCache(self.local_storage)
         self.cache.catalog_source = _sample_metadata_catalog_source()
 
+    def test_add_does_not_swallow_exceptions(self):
+        original_add = self.cache.graph.add
+        self.cache.graph.add = always_throw(IOError)
+        try:
+            with self.assertRaises(IOError):
+                self.test_populate()
+        finally:
+            self.cache.graph.add = original_add
+
 
 def _sample_metadata_catalog_source():
     module = os.path.dirname(sys.modules['tests'].__file__)