Merge pull request #4082 from GeotrekCE/impr_sources_picto_to_aggreg

💫 [IMPR] Add sources websites and pictograms to Aggregator (refs #3569)

docs/changelog.rst

@@ -5,6 +5,10 @@ CHANGELOG
 2.105.1+dev (XXXX-XX-XX)
 ------------------------
 
+**Improvements**
+
+- Add sources websites and pictograms to Aggregator (#3569)
+
 
 2.105.1 (2024-05-07)
 --------------------

geotrek/common/parsers.py

@@ -39,7 +39,7 @@
 from modeltranslation.utils import build_localized_fieldname
 
 from geotrek.authent.models import default_structure
-from geotrek.common.models import FileType, Attachment, License
+from geotrek.common.models import FileType, Attachment, License, RecordSource
 from geotrek.common.utils.parsers import add_http_prefix
 from geotrek.common.utils.translation import get_translated_fields
 
@@ -1350,6 +1350,10 @@ def __init__(self, all_datas=None, create_categories=None, provider=None, mappin
                 raise ImproperlyConfigured(f"{category} is not configured in categories_keys_api_v2")
         self.creator, created = get_user_model().objects.get_or_create(username='import', defaults={'is_active': False})
 
+        # Update sources if applicable
+        if "source" in self.url_categories.keys():
+            self.get_sources_extra_fields()
+
     def replace_mapping(self, label, route):
         for key, list_map in self.mapping.get(route, {}).items():
             if label in list_map:
@@ -1477,6 +1481,34 @@ def next_row(self):
 
             self.next_url = self.root['next']
 
+    def get_sources_extra_fields(self):
+        response = self.request_or_retry(f"{self.url}/api/v2/source/")
+        create = self.field_options['source'].get("create", False)
+        if create:
+            for result in response.json()['results']:
+                name = result['name']
+                pictogram_url = result['pictogram']
+                website = result['website']
+                source, created = RecordSource.objects.update_or_create(**{'name': name}, defaults={'website': website})
+                if created:
+                    self.add_warning(_(f"Source '{name}' did not exist in Geotrek-Admin and was automatically created"))
+                if not pictogram_url and source.pictogram:
+                    source.pictogram.delete()
+                elif pictogram_url:
+                    pictogram_filename = os.path.basename(pictogram_url)
+                    if not source.pictogram or pictogram_filename != source.pictogram.file.name:
+                        try:
+                            response = self.request_or_retry(pictogram_url)
+                        except (DownloadImportError, requests.exceptions.ConnectionError):
+                            self.add_warning(_("Failed to download '{url}'").format(url=pictogram_url))
+                            return
+                        if response.status_code != requests.codes.ok:
+                            self.add_warning(_("Failed to download '{url}'").format(url=pictogram_url))
+                            return
+                        if response.content:
+                            pictogram_file = ContentFile(response.content)
+                            source.pictogram.save(pictogram_filename, pictogram_file)
+
 
 class ApidaeBaseParser(Parser):
     """Parser to import "anything" from APIDAE"""

geotrek/common/tests/test_parsers.py

@@ -16,15 +16,15 @@
 from requests import Response
 
 from geotrek.authent.tests.factories import StructureFactory
-from geotrek.common.models import Attachment, FileType, Organism, Theme
+from geotrek.common.models import Attachment, FileType, Organism, RecordSource, Theme
 from geotrek.common.parsers import (AttachmentParserMixin, DownloadImportError,
                                     ExcelParser, GeotrekAggregatorParser,
                                     GeotrekParser, OpenSystemParser,
                                     TourInSoftParser, TourismSystemParser,
                                     ValueImportError, XmlParser)
 from geotrek.common.tests.factories import ThemeFactory
 from geotrek.common.tests.mixins import GeotrekParserTestMixin
-from geotrek.common.utils.testdata import get_dummy_img
+from geotrek.common.utils.testdata import SVG_FILE, get_dummy_img
 from geotrek.trekking.models import POI, Trek
 from geotrek.trekking.parsers import GeotrekTrekParser
 from geotrek.trekking.tests.factories import TrekFactory
@@ -822,6 +822,7 @@ def test_geotrek_aggregator_parser(self, mocked_head, mocked_get):
             ('trekking', 'trek_network.json'),
             ('trekking', 'trek_label.json'),
             ('trekking', 'sources.json'),
+            ('trekking', 'sources.json'),
             ('trekking', 'structure.json'),
             ('trekking', 'poi_type.json'),
             ('trekking', 'trek_ids.json'),
@@ -846,6 +847,7 @@ def test_geotrek_aggregator_parser(self, mocked_head, mocked_get):
             ('trekking', 'trek_network.json'),
             ('trekking', 'trek_label.json'),
             ('trekking', 'sources.json'),
+            ('trekking', 'sources.json'),
             ('trekking', 'structure.json'),
             ('trekking', 'poi_type.json'),
             ('trekking', 'trek_ids.json'),
@@ -884,3 +886,85 @@ def test_geotrek_aggregator_parser(self, mocked_head, mocked_get):
         call_command('import', 'geotrek.common.parsers.GeotrekAggregatorParser', filename=filename, verbosity=2,
                      stdout=output)
         self.assertEqual(1, Trek.objects.get(name="Boucle du Pic des Trois Seigneurs").information_desks.count())
+
+
+class GeotrekTrekTestSourcesParser(GeotrekTrekParser):
+    url = "https://test.fr"
+    model = Trek
+    url_categories = {
+        'source': 'source'
+    }
+    field_options = {
+        'source': {'create': True}
+    }
+    constant_fields = {'structure': settings.DEFAULT_STRUCTURE_NAME}
+
+
+class GeotrekAggregatorSourcesTests(TestCase):
+
+    def mocked_responses(self, url):
+        class MockResponse:
+            def __init__(self, mock_time, status_code):
+                self.content = SVG_FILE
+                self.mock_time = mock_time
+                self.mock_json_order = [
+                    # First time
+                    ('trekking', 'sources.json'),
+                    ('trekking', 'sources.json'),
+                    # # Second time
+                    ('trekking', 'sources.json'),
+                    ('trekking', 'sources_updated.json'),
+                    # Third time
+                    ('trekking', 'sources.json'),
+                    ('trekking', 'sources_error_1.json'),
+                    ('trekking', 'iwillthrowerror.json"'),
+                    # Fourth time
+                    ('trekking', 'sources.json'),
+                    ('trekking', 'sources_error_2.json'),
+                ]
+                self.status_code = status_code
+
+            def json(self):
+                filename = os.path.join('geotrek', self.mock_json_order[self.mock_time][0], 'tests', 'data', 'geotrek_parser_v2',
+                                        self.mock_json_order[self.mock_time][1])
+                with open(filename, 'r') as f:
+                    return json.load(f)
+
+        if self.mock_time == 6:
+            self.mock_time += 1
+            raise requests.exceptions.ConnectionError
+        status_code = 200
+        if self.mock_time == 9:
+            status_code = 404
+            self.mock_time += 1
+        mocked_response = MockResponse(self.mock_time, status_code)
+        if ".png" not in url:
+            self.mock_time += 1
+        return mocked_response
+
+    @mock.patch('requests.get')
+    @mock.patch('geotrek.common.parsers.GeotrekParser.request_or_retry')
+    @mock.patch('geotrek.common.parsers.GeotrekParser.add_warning')
+    def test_sources_extra_fields_parsing(self, mocked_add_warning, mocked_request_or_retry, mocked_get):
+        self.mock_time = 0
+        mocked_request_or_retry.side_effect = self.mocked_responses
+
+        # Test created
+        GeotrekTrekTestSourcesParser()
+        s = RecordSource.objects.get(name="Parc national des Ecrins")
+        self.assertEqual(s.website, "https://www.ecrins-parcnational.fr")
+        self.assertEqual(s.pictogram.file.name.split('/')[-1], "pnecrins.png")
+
+        # Test updated
+        GeotrekTrekTestSourcesParser()
+        s.refresh_from_db()
+        self.assertEqual(s.website, "")
+        self.assertEqual(s.pictogram, "")
+
+        # Test Connection Error
+        GeotrekTrekTestSourcesParser()
+        mocked_add_warning.assert_called_with("Failed to download 'https://geotrek-admin.ecrins-parcnational.fr/media/upload/iwillthrowerror.png'")
+
+        # Test bad response status
+        GeotrekTrekTestSourcesParser()
+        mocked_add_warning.assert_called_with("Failed to download 'https://geotrek-admin.ecrins-parcnational.fr/media/upload/iwillthrowerroragain.png'")

geotrek/outdoor/tests/test_parsers.py

@@ -31,11 +31,13 @@ def test_geotrek_aggregator_parser_mapping(self, mocked_head, mocked_get):
                                 ('outdoor', 'label.json'),
                                 ('outdoor', 'source.json'),
                                 ('outdoor', 'organism.json'),
+                                ('outdoor', 'source.json'),
                                 ('outdoor', 'structure.json'),
                                 ('outdoor', 'theme.json'),
                                 ('outdoor', 'label.json'),
                                 ('outdoor', 'source.json'),
                                 ('outdoor', 'organism.json'),
+                                ('outdoor', 'source.json'),
                                 ('outdoor', 'outdoor_site_ids.json'),
                                 ('outdoor', 'outdoor_sector.json'),
                                 ('outdoor', 'outdoor_practice.json'),
@@ -90,11 +92,13 @@ def test_create_sites_and_courses(self, mocked_head, mocked_get):
                                 ('outdoor', 'label.json'),
                                 ('outdoor', 'source.json'),
                                 ('outdoor', 'organism.json'),
+                                ('outdoor', 'source.json'),
                                 ('outdoor', 'structure.json'),
                                 ('outdoor', 'theme.json'),
                                 ('outdoor', 'label.json'),
                                 ('outdoor', 'source.json'),
                                 ('outdoor', 'organism.json'),
+                                ('outdoor', 'source.json'),
                                 ('outdoor', 'outdoor_site_ids.json'),
                                 ('outdoor', 'outdoor_sector.json'),
                                 ('outdoor', 'outdoor_practice.json'),
@@ -243,11 +247,13 @@ def test_create_sites_and_courses_with_wrong_children(self, mocked_head, mocked_
                                 ('outdoor', 'label.json'),
                                 ('outdoor', 'source.json'),
                                 ('outdoor', 'organism.json'),
+                                ('outdoor', 'source.json'),
                                 ('outdoor', 'structure.json'),
                                 ('outdoor', 'theme.json'),
                                 ('outdoor', 'label.json'),
                                 ('outdoor', 'source.json'),
                                 ('outdoor', 'organism.json'),
+                                ('outdoor', 'source.json'),
                                 ('outdoor', 'outdoor_site_ids.json'),
                                 ('outdoor', 'outdoor_sector.json'),
                                 ('outdoor', 'outdoor_practice.json'),

geotrek/tourism/tests/test_parsers.py

@@ -933,6 +933,7 @@ def test_create(self, mocked_head, mocked_get):
                                 ('tourism', 'touristiccontent_category.json'),
                                 ('tourism', 'touristiccontent_themes.json'),
                                 ('tourism', 'sources.json'),
+                                ('tourism', 'sources.json'),
                                 ('tourism', 'touristiccontent_category.json'),
                                 ('tourism', 'touristiccontent_ids.json'),
                                 ('tourism', 'touristiccontent.json')]
@@ -964,6 +965,7 @@ def test_create_create_categories(self, mocked_head, mocked_get):
                                 ('tourism', 'touristiccontent_category.json'),
                                 ('tourism', 'touristiccontent_themes.json'),
                                 ('tourism', 'sources.json'),
+                                ('tourism', 'sources.json'),
                                 ('tourism', 'touristiccontent_category.json'),
                                 ('tourism', 'touristiccontent_ids.json'),
                                 ('tourism', 'touristiccontent.json')]
@@ -999,6 +1001,7 @@ def test_create(self, mocked_head, mocked_get):
         self.mock_json_order = [('tourism', 'structure.json'),
                                 ('tourism', 'touristicevent_type.json'),
                                 ('tourism', 'sources.json'),
+                                ('tourism', 'sources.json'),
                                 ('tourism', 'touristicevent_ids.json'),
                                 ('tourism', 'touristicevent.json')]
 

geotrek/trekking/tests/data/geotrek_parser_v2/sources.json

@@ -6,7 +6,7 @@
         {
             "id": 1,
             "name": "Parc national des Ecrins",
-            "pictogram": "https://geotrek-admin.ecrins-parcnational.fr/media/upload/pnecrins.jpg",
+            "pictogram": "https://geotrek-admin.ecrins-parcnational.fr/media/upload/pnecrins.png",
             "website": "https://www.ecrins-parcnational.fr"
         },
         {

geotrek/trekking/tests/data/geotrek_parser_v2/sources_error_1.json

@@ -0,0 +1,19 @@
+{
+    "count": 8,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 1,
+            "name": "Parc national des Ecrins",
+            "pictogram": "https://geotrek-admin.ecrins-parcnational.fr/media/upload/iwillthrowerror.png",
+            "website": ""
+        },
+        {
+            "id": 2,
+            "name": "Une source numero 2",
+            "pictogram": "https://geotrek-admin.ecrins-parcnational.fr/media/upload/logo-mtcv.png",
+            "website": ""
+        }
+    ]
+}

geotrek/trekking/tests/data/geotrek_parser_v2/sources_error_2.json

@@ -0,0 +1,19 @@
+{
+    "count": 8,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 1,
+            "name": "Parc national des Ecrins",
+            "pictogram": "https://geotrek-admin.ecrins-parcnational.fr/media/upload/iwillthrowerroragain.png",
+            "website": ""
+        },
+        {
+            "id": 2,
+            "name": "Une source numero 2",
+            "pictogram": "https://geotrek-admin.ecrins-parcnational.fr/media/upload/logo-mtcv.png",
+            "website": ""
+        }
+    ]
+}

geotrek/trekking/tests/data/geotrek_parser_v2/sources_updated.json

@@ -0,0 +1,19 @@
+{
+    "count": 8,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 1,
+            "name": "Parc national des Ecrins",
+            "pictogram": null,
+            "website": ""
+        },
+        {
+            "id": 2,
+            "name": "Une source numero 2",
+            "pictogram": "https://geotrek-admin.ecrins-parcnational.fr/media/upload/logo-mtcv.png",
+            "website": ""
+        }
+    ]
+}