Clear datasets only after successful request

.circleci/config.yml

@@ -28,6 +28,7 @@ jobs:
           command: |
             virtualenv venv
             source venv/bin/activate
+            pip install --force-reinstall setuptools==66.1.1
             pip install -r requirements/develop.pip
             pip install -e .
       - save_cache:

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Current (in progress)
 
 - Replace mongo legacy image in CI [#2](https://github.com/opendatateam/udata-transport/pull/2)
+- Clear datasets only after successful request [#3](https://github.com/opendatateam/udata-transport/pull/3)
 
 ## 1.0.0 (2022-03-31)
 

requirements/develop.pip

@@ -1,6 +1,5 @@
 -e .[test]
 flake8==3.7.8
-flit==3.6.0
-invoke==1.3.0
-pytest-cov==2.7.1
+invoke==1.7.3
+pytest-cov==4.0.0
 twine==3.3.0

requirements/test.pip

@@ -1,6 +1,6 @@
-pytest==4.6.3
-pytest-flask==0.15.0
-pytest-sugar==0.9.2
-requests-mock==1.7.0
-mock==3.0.5
-pytest-mock==2.0.0
+pytest==7.3.1
+pytest-flask==1.2.0
+pytest-sugar==0.9.7
+requests-mock==1.10.0
+mock==5.0.1
+pytest-mock==3.10.0

tasks.py

@@ -64,7 +64,7 @@ def test(ctx, report=False):
 @task
 def cover(ctx, html=False):
     '''Run tests suite with coverage'''
-    cmd = 'pytest --cov udata_recommendations --cov-report term'
+    cmd = 'pytest --cov udata_transport --cov-report term'
     if html:
         cmd = ' '.join((cmd, '--cov-report html:reports/cover'))
     with ctx.cd(ROOT):
@@ -77,7 +77,7 @@ def qa(ctx):
     header(qa.__doc__)
     with ctx.cd(ROOT):
         info('Python Static Analysis')
-        flake8_results = ctx.run('flake8 udata_recommendations', pty=True, warn=True)
+        flake8_results = ctx.run('flake8 udata_transport', pty=True, warn=True)
         if flake8_results.failed:
             error('There is some lints to fix')
         else:

tests/test_tasks.py

@@ -43,3 +43,22 @@ def test_map_transport_datasets(self, mock_response):
 
         assert ds1.extras['transport:url'] == 'https://transport.data.gouv.fr/datasets/horaires-theoriques-et-temps-reel-des-navettes-hivernales-de-lalpe-dhuez-gtfs-gtfs-rt'
         assert ds2.extras['transport:url'] == 'https://transport.data.gouv.fr/datasets/horaires-theoriques-et-temps-reel-du-reseau-lr-11-lalouvesc-tournon-st-felicien-gtfs-gtfs-rt'
+
+    @pytest.mark.options(TRANSPORT_DATASETS_URL='http://local.test/api/datasets')
+    def test_map_transport_datasets_fail(self, mock_response):
+        '''
+        We should not erase existing transport:url extras if the job fails
+        '''
+        ds1 = DatasetFactory(id='61fd29da29ea95c7bc0e1211',
+                             extras={'transport:url': 'dummy'})
+        ds2 = DatasetFactory(id='5f23d4b3d39755210a04a99c')
+
+        with requests_mock.Mocker() as m:
+            m.get('http://local.test/api/datasets', status_code=500)
+            map_transport_datasets()
+
+        ds1.reload()
+        ds2.reload()
+
+        assert ds1.extras['transport:url'] == 'dummy'
+        assert 'transport:url' not in ds2.extras

udata_transport/__init__.py

@@ -1,6 +1,6 @@
-'''
+"""
 Udata Transport
-'''
+"""
 
 __version__ = '1.0.1.dev'
 __description__ = 'Transport.data.gouv.fr mapping to Udata'

udata_transport/settings.py

@@ -1,6 +1,6 @@
-'''
+"""
 Default settings for udata-transport
-'''
+"""
 
 
 TRANSPORT_DATASETS_URL = 'https://transport.data.gouv.fr/api/datasets'

udata_transport/tasks.py

@@ -14,13 +14,17 @@ def process_dataset(dataset):
     target_dataset.save()
 
 
-@job("map-transport-datasets")
-def map_transport_datasets(self):
-    datasets = Dataset.objects(**{'extras__transport:url__exists': True}).no_cache().timeout(False)
+def clear_datasets():
+    datasets = Dataset.objects(
+        **{'extras__transport:url__exists': True}
+    ).no_cache().timeout(False)
     for dataset in datasets:
         dataset.extras.pop('transport:url', None)
         dataset.save()
 
+
+@job('map-transport-datasets')
+def map_transport_datasets(self):
     source = current_app.config.get('TRANSPORT_DATASETS_URL', None)
     if not source:
         error('TRANSPORT_DATASETS_URL variable must be set.')
@@ -31,6 +35,7 @@ def map_transport_datasets(self):
         error('Remote platform unreachable.')
         return
     results_list = response.json()
+    clear_datasets()
     for dataset in results_list:
         process_dataset(dataset)
     success(f'Done. {len(results_list)} datasets mapped to transport')

udata_transport/views.py

@@ -1,4 +1,4 @@
-from flask import Blueprint, current_app
+from flask import Blueprint
 
 from udata_front import theme
 from udata.frontend import template_hook