Add a dict of URIs to replace in a RDF graph at harvest time

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Add `business_number_id` metadata for organizations [#2871](https://github.com/opendatateam/udata/pull/2871)
 - Return 403 when posting comment on discussion closed [#2881](https://github.com/opendatateam/udata/pull/2881)
 - Ensure rdf parsed frequency is lowercase [#2883](https://github.com/opendatateam/udata/pull/2883)
+- Add a dict of URIs to replace in a RDF graph at harvest time [#2884](https://github.com/opendatateam/udata/pull/2884)
 
 ## 6.1.6 (2023-07-19)
 

udata/harvest/backends/dcat.py

@@ -35,6 +35,12 @@
     (HYDRA.PagedCollection, HYDRA.nextPage)
 )
 
+# Useful to patch essential failing URIs
+URIS_TO_REPLACE = {
+    # See https://github.com/etalab/data.gouv.fr/issues/1151
+    'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld': 'https://gist.githubusercontent.com/maudetes/f019586185d6f59dcfb07f97148a1973/raw/585c3c7bf602b5a4e635b137257d0619792e2c1f/gistfile1.txt'  # noqa
+}
+
 
 def extract_graph(source, target, node, specs):
     for p, o in source.predicate_objects(node):
@@ -80,7 +86,10 @@ def parse_graph(self, url, fmt) -> List[Graph]:
         page = 0
         while url:
             subgraph = Graph(namespace_manager=namespace_manager)
-            subgraph.parse(data=requests.get(url).text, format=fmt)
+            data = requests.get(url).text
+            for old_uri, new_uri in URIS_TO_REPLACE.items():
+                data = data.replace(old_uri, new_uri)
+            subgraph.parse(data=data, format=fmt)
 
             url = None
             for cls, prop in KNOWN_PAGINATION:

udata/harvest/tests/test_dcat_backend.py

@@ -10,6 +10,7 @@
 from udata.core.dataset.factories import LicenseFactory
 
 from .factories import HarvestSourceFactory
+from ..backends.dcat import URIS_TO_REPLACE
 from .. import actions
 
 log = logging.getLogger(__name__)
@@ -333,7 +334,7 @@ def test_sigoreme_xml_catalog(self, rmock):
         assert 'geodesy' in dataset.tags  # support dcat:theme
         assert dataset.license.id == 'fr-lo'
         assert len(dataset.resources) == 1
-        assert dataset.description.startswith('Data from the \'National network')
+        assert dataset.description.startswith("Data from the 'National network")
         assert dataset.harvest is not None
         assert dataset.harvest.dct_identifier == '0437a976-cff1-4fa6-807a-c23006df2f8f'
         assert dataset.harvest.remote_id == '0437a976-cff1-4fa6-807a-c23006df2f8f'
@@ -383,3 +384,27 @@ def test_unable_to_detect_format(self, rmock):
         error = job.errors[0]
         expected = 'Unable to detect format from extension or mime type'
         assert error.message == expected
+
+    def test_use_replaced_uris(self, rmock, mocker):
+        mocker.patch.dict(
+            URIS_TO_REPLACE,
+            {'http://example.org/this-url-does-not-exist': 'https://json-ld.org/contexts/person.jsonld'}
+        )
+        url = DCAT_URL_PATTERN.format(path='', domain=TEST_DOMAIN)
+        rmock.get(url, json={
+            '@context': 'http://example.org/this-url-does-not-exist',
+            '@type': 'dcat:Catalog',
+            'dataset': []
+        })
+        rmock.head(url, headers={'Content-Type': 'application/json'})
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat',
+                                      url=url,
+                                      organization=org)
+        actions.run(source.slug)
+
+        source.reload()
+
+        job = source.get_last_job()
+        assert len(job.items) == 0
+        assert job.status == 'done'