From 29a47ca8d1df317b61f00e0a4b9993acf39eec3f Mon Sep 17 00:00:00 2001 From: esir Date: Thu, 4 Feb 2021 13:30:14 +0300 Subject: [PATCH 1/4] Adds a management command to reverse geo code cities for nodes with no cities --- requirements.txt | 4 +++- .../management/commands/add_city_names.py | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 sensorsafrica/management/commands/add_city_names.py diff --git a/requirements.txt b/requirements.txt index fdca0c5..7ad0f1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,6 @@ celery-slack==0.3.0 urllib3<1.25,>=1.21.1 #requests 2.21.0 -django-cors-headers==3.0.2 \ No newline at end of file +django-cors-headers==3.0.2 + +geopy==2.1.0 diff --git a/sensorsafrica/management/commands/add_city_names.py b/sensorsafrica/management/commands/add_city_names.py new file mode 100644 index 0000000..71d135d --- /dev/null +++ b/sensorsafrica/management/commands/add_city_names.py @@ -0,0 +1,24 @@ +from django.core.management import BaseCommand +from django.db.models import Q + +from geopy.geocoders import Nominatim + +from feinstaub.sensors.models import Node + + +class Command(BaseCommand): + help = "Adds city names to SensorLocation by geo reversing the SensorLocation longitude and latitude." + + def handle(self, *args, **options): + geolocator = Nominatim(user_agent="sensors-api") + all_nodes = Node.objects.filter(Q(location__city=None) | Q(location__city='')) + cities = [] + for node in all_nodes: + try: + location = geolocator.reverse(f"{node.location.latitude}, {node.location.longitude}") + except Exception: + # Nodes with location like Soul Buoy raises exceptions + continue + city = location.raw['address'].get('city') + node.location.city = city + node.save() From 461d2400d1a7211323a9864c79a88f15fd301c1b Mon Sep 17 00:00:00 2001 From: esir Date: Thu, 4 Feb 2021 13:30:45 +0300 Subject: [PATCH 2/4] Return cities in meta api endpoint --- sensorsafrica/api/v2/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index b334231..76e9735 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -421,6 +421,7 @@ def meta_data(request): database_size = get_database_size() database_last_updated = get_database_last_updated() sensors_locations = get_sensors_locations() + sensors_cities = get_sensors_cities() return Response({ "sensor_networks": get_sensors_networks(), @@ -428,6 +429,7 @@ def meta_data(request): "sensors_count": sensors_count, "sensor_data_count": sensor_data_count, "sensors_locations": sensors_locations, + "sensors_cities": sensors_cities, "database_size": database_size[0], "database_last_updated": database_last_updated, }) @@ -443,6 +445,10 @@ def get_sensors_locations(): sensor_locations = SensorLocation.objects.filter(country__isnull=False).values_list('country', flat=True) return sorted(set(sensor_locations)) +def get_sensors_cities(): + sensor_cities = Node.objects.filter(location__city__isnull=False).values_list('location__city', flat=True) + return sorted(set(sensor_cities)) + def get_database_size(): with connection.cursor() as c: c.execute(f"SELECT pg_size_pretty(pg_database_size('{connection.settings_dict['NAME']}'))") From 9db1d54292ed9c18e580b835ff31b218df35e622 Mon Sep 17 00:00:00 2001 From: esir Date: Thu, 4 Feb 2021 14:05:02 +0300 Subject: [PATCH 3/4] Update response to use sensors_countries --- sensorsafrica/api/v2/views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index 76e9735..cccb1b6 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -420,7 +420,7 @@ def meta_data(request): database_size = get_database_size() database_last_updated = get_database_last_updated() - sensors_locations = get_sensors_locations() + sensors_countries = get_sensors_countries() sensors_cities = get_sensors_cities() return Response({ @@ -428,7 +428,7 @@ def meta_data(request): "nodes_count": nodes_count, "sensors_count": sensors_count, "sensor_data_count": sensor_data_count, - "sensors_locations": sensors_locations, + "sensors_countries": sensors_countries, "sensors_cities": sensors_cities, "database_size": database_size[0], "database_last_updated": database_last_updated, @@ -441,9 +441,9 @@ def get_sensors_networks(): networks.append("sensors.AFRICA") return {"networks": networks, "count": len(networks)} -def get_sensors_locations(): - sensor_locations = SensorLocation.objects.filter(country__isnull=False).values_list('country', flat=True) - return sorted(set(sensor_locations)) +def get_sensors_countries(): + sensors_countries = SensorLocation.objects.filter(country__isnull=False).values_list('country', flat=True) + return sorted(set(sensors_countries)) def get_sensors_cities(): sensor_cities = Node.objects.filter(location__city__isnull=False).values_list('location__city', flat=True) From 435102c4284b5b0b75d8540fe661f494aa9908ae Mon Sep 17 00:00:00 2001 From: esir Date: Thu, 4 Feb 2021 14:07:32 +0300 Subject: [PATCH 4/4] Remove un used code --- sensorsafrica/management/commands/add_city_names.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensorsafrica/management/commands/add_city_names.py b/sensorsafrica/management/commands/add_city_names.py index 71d135d..3ceaf2e 100644 --- a/sensorsafrica/management/commands/add_city_names.py +++ b/sensorsafrica/management/commands/add_city_names.py @@ -12,7 +12,7 @@ class Command(BaseCommand): def handle(self, *args, **options): geolocator = Nominatim(user_agent="sensors-api") all_nodes = Node.objects.filter(Q(location__city=None) | Q(location__city='')) - cities = [] + for node in all_nodes: try: location = geolocator.reverse(f"{node.location.latitude}, {node.location.longitude}")