Merge 1de4177 into c5e975f

OSMCha · Aug 4, 2020 · 0d81418 · 0d81418
2 parents c5e975f + 1de4177
commit 0d81418
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 6 deletions.
diff --git a/osmchadjango/changeset/filters.py b/osmchadjango/changeset/filters.py
@@ -238,6 +238,37 @@ class ChangesetFilter(GeoFilterSet):
         help_text="""Filter changesets that have ALL the Tags of a list. Inform
             the Tags ids separated by commas."""
         )
+    metadata = filters.CharFilter(
+        field_name='metadata',
+        method='filter_metadata',
+        help_text="""Filter changesets by the metadata fields."""
+        )
+
+    def filter_metadata(self, queryset, name, value):
+        values = [
+            [i.strip() for i in t.split('=')]  # remove leading and ending spaces
+            for t in value.split(',')
+            if len(t.split('=')) == 2
+            ]
+        for query in values:
+            if '__' in query[0]:
+                # handle both int values and other lookup options like __exact or __contains
+                key = 'metadata__{}'.format(
+                    query[0].replace('__min', '__gte').replace('__max', '__lte')
+                    )
+                try:
+                    value = int(query[1])
+                except ValueError:
+                    value = query[1]
+            elif query[1] == '*':
+                key = 'metadata__has_key'
+                value = query[0]
+            else:
+                # default option is to use the icontains condition
+                key = key = 'metadata__{}__icontains'.format(query[0])
+                value = query[1]
+            queryset = queryset.filter(**{key: value})
+        return queryset
 
     def filter_whitelist(self, queryset, name, value):
         if value:
@@ -263,7 +294,7 @@ def get_username_from_teams(self, teams):
         users = []
         for i in teams.values_list('users', flat=True):
             values = i
-            if type(values) in[str, bytes, bytearray]:
+            if type(values) in [str, bytes, bytearray]:
                 values = json.loads(values)
             for e in values:
                 users.append(e.get('username'))

diff --git a/osmchadjango/changeset/migrations/0051_changeset_metadata.py b/osmchadjango/changeset/migrations/0051_changeset_metadata.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.2.13 on 2020-08-04 09:59
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('changeset', '0050_auto_20181008_1001'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='changeset',
+            name='metadata',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+        ),
+    ]
diff --git a/osmchadjango/changeset/models.py b/osmchadjango/changeset/models.py
@@ -82,6 +82,7 @@ class Changeset(models.Model):
         User, on_delete=models.SET_NULL, null=True, blank=True, db_index=True
         )
     check_date = models.DateTimeField(null=True, blank=True)
+    metadata = JSONField(default=dict)
 
     def __str__(self):
         return '%s' % self.id

diff --git a/osmchadjango/changeset/tests/modelfactories.py b/osmchadjango/changeset/tests/modelfactories.py
@@ -43,6 +43,7 @@ class Meta:
         (-71.0048652, 44.2430624), (-71.0646843, 44.2430624),
         (-71.0646843, 44.2371354)
         ])
+    metadata = {'changesets_count': 99, 'host': 'https://ideditor.netlify.app'}
 
 
 class SuspectChangesetFactory(ChangesetFactory):

diff --git a/osmchadjango/changeset/tests/test_changeset_views.py b/osmchadjango/changeset/tests/test_changeset_views.py
@@ -225,6 +225,18 @@ def test_mapping_team_filter(self):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data['count'], 1)
 
+    def test_metadata_filters(self):
+        """Test filters by metadata field in the changeset list view.
+        """
+        self.client.login(username=self.user.username, password='password')
+
+        response = self.client.get(self.url, {'metadata': 'changesets_count__min=100'})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 0)
+        response = self.client.get(self.url, {'metadata': 'changesets_count__min=99', 'is_suspect': True})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 26)
+
     def test_csv_renderer(self):
         self.assertIn(
             PaginatedCSVRenderer,

diff --git a/osmchadjango/changeset/tests/test_filters.py b/osmchadjango/changeset/tests/test_filters.py
@@ -27,17 +27,20 @@ def setUp(self):
             user='suspect_user',
             uid='343',
             source='Bing',
-            imagery_used='Bing'
+            imagery_used='Bing',
+            metadata={'changesets_count': 50, 'host': 'https://www.openstreetmap.org/edit'}
             )
         self.harmful_changeset = HarmfulChangesetFactory(
             check_user=self.user,
             editor='JOSM 1.5',
             powerfull_editor=True,
-            imagery_used='Mapbox, Mapillary'
+            imagery_used='Mapbox, Mapillary',
+            metadata={'changesets_count': 500, 'host': 'https://www.openstreetmap.org/edit'}
             )
         self.good_changeset = GoodChangesetFactory(
             check_user=self.user_2,
-            source='Mapbox'
+            source='Mapbox',
+            metadata={'changesets_count': 10, 'locale': 'en'}
             )
         self.reason_1 = SuspicionReasonsFactory(name='possible import')
         self.reason_1.changesets.add(self.suspect_changeset)
@@ -322,6 +325,34 @@ def test_mapping_team_filter(self):
             1
             )
 
+    def test_metadata_filter(self):
+        self.assertEqual(
+            ChangesetFilter({'metadata': 'changesets_count__min=101,host=openstreetmap.org'}).qs.count(),
+            1
+            )
+        self.assertEqual(
+            ChangesetFilter({'metadata': 'changesets_count__min=101 , host=openstreetmap.org '}).qs.count(),
+            1
+            )
+        self.assertEqual(
+            ChangesetFilter({'metadata': 'changesets_count__max=99'}).qs.count(),
+            3
+            )
+        self.assertEqual(ChangesetFilter({'metadata': 'host=*'}).qs.count(), 3)
+        self.assertEqual(ChangesetFilter({'metadata': 'host__exact=.org'}).qs.count(), 0)
+        self.assertEqual(
+            ChangesetFilter({'metadata': 'host__exact=https://www.openstreetmap.org/edit'}).qs.count(),
+            2
+            )
+        self.assertEqual(ChangesetFilter({'metadata': 'locale=*'}).qs.count(), 1)
+        self.assertEqual(ChangesetFilter({'metadata': 'locale=* '}).qs.count(), 1)
+        self.assertEqual(ChangesetFilter({'metadata': 'locale=en'}).qs.count(), 1)
+        self.assertEqual(ChangesetFilter({'metadata': 'locale=pt'}).qs.count(), 0)
+        self.assertEqual(ChangesetFilter({'metadata': 'wrongtag=pt'}).qs.count(), 0)
+        self.assertEqual(ChangesetFilter({'metadata': 'wrongtag=*'}).qs.count(), 0)
+        self.assertEqual(ChangesetFilter({'metadata': 'wrongtag'}).qs.count(), 4)
+        self.assertEqual(ChangesetFilter({'metadata': 'wrongtag__min=abc'}).qs.count(), 0)
+
 
 class TestChangesetAreaLowerThan(TestCase):
     def setUp(self):

diff --git a/osmchadjango/changeset/tests/test_models.py b/osmchadjango/changeset/tests/test_models.py
@@ -133,7 +133,6 @@ def test_tag(self):
     def test_empty_new_feature_field(self):
         self.assertEqual(self.changeset.new_features, [])
 
-
     def test_new_feature_field(self):
         json_content = [
             {"osm_id": 123, "url": "node-123", "reasons": [1, 2], },
@@ -146,6 +145,12 @@ def test_new_feature_field(self):
         self.assertEqual(Changeset.objects.all().count(), 2)
         self.assertEqual(changeset.new_features, json_content)
 
+    def test_metadata_field(self):
+        self.assertEqual(
+            self.changeset.metadata,
+            {'changesets_count': 99, 'host': 'https://ideditor.netlify.app'}
+            )
+
 
 class TestChangesetModelOrdering(TestCase):
     def setUp(self):

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -42,7 +42,7 @@ redis>=2.10.5
 
 # Your custom requirements go here
 PyYAML==5.3.1
-osmcha==0.6.0
+osmcha==0.7.0
 
 # git python is required by the frontend management command
 GitPython==3.1.7