additional_data: geo_lookup: Add new data sources

Adds data sources for regions, countries, upper tier local authorities and combined authority. Fixes #71 Co-authored-by: Michael Wood <michael.wood@opendataservices.coop>
ThreeSixtyGiving · Nov 10, 2021 · 73a0823 · 73a0823
1 parent 6efda64
commit 73a0823
Show file tree

Hide file tree

Showing 6 changed files with 269 additions and 3 deletions.
diff --git a/datastore/additional_data/sources/geo_lookup.py b/datastore/additional_data/sources/geo_lookup.py
@@ -47,6 +47,26 @@ class GeoLookupSource(object):
                 "lad20nm": "ladnm",
             },
         },
+        "rgn": {
+            "url_lookup": "https://raw.githubusercontent.com/drkane/geo-lookups/master/rgn_all_codes.csv",
+            "field_areacode": "rgncd",
+            "field_areaname": "rgnnm",
+        },
+        "ctry": {
+            "url_lookup": "https://raw.githubusercontent.com/drkane/geo-lookups/master/ctry_all_codes.csv",
+            "field_areacode": "ctrycd",
+            "field_areaname": "ctrynm",
+        },
+        "utla": {
+            "url_lookup": "https://raw.githubusercontent.com/drkane/geo-lookups/master/utla_all_codes.csv",
+            "field_areacode": "utlacd",
+            "field_areaname": "utlanm",
+        },
+        "cuath": {
+            "url_lookup": "https://raw.githubusercontent.com/drkane/geo-lookups/master/cauth_all_codes.csv",
+            "field_areacode": "cauthcd",
+            "field_areaname": "cauthnm",
+        },
     }
 
     def __init__(self):
@@ -65,6 +85,7 @@ def clean_fields(row):
                 yield field, value
 
         r = requests.get(areadata.get("url_lookup"), stream=True)
+        r.encoding = "utf-8-sig"
         area_csv = io.StringIO(r.text)
         reader = csv.DictReader(area_csv)
         for r in reader:
@@ -75,6 +96,7 @@ def clean_fields(row):
 
         if areadata.get("url_latlong"):
             r = requests.get(areadata.get("url_latlong"), stream=True)
+            r.encoding = "utf-8-sig"
             area_csv = io.StringIO(r.text)
             reader = csv.DictReader(area_csv)
             for r in reader:

diff --git a/datastore/tests/files/geolookups/cauth_all_codes.csv b/datastore/tests/files/geolookups/cauth_all_codes.csv
@@ -0,0 +1,11 @@
+CAUTHCD,CAUTHNM,RGNCD,RGNNM,CTRYCD,CTRYNM,EWCD,EWNM,GBCD,GBNM,UKCD,UKNM
+E47000001,Greater Manchester,E12000002,North West,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000002,Sheffield City Region,E12000003,Yorkshire and The Humber,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000003,West Yorkshire,E12000003,Yorkshire and The Humber,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000004,Liverpool City Region,E12000002,North West,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000006,Tees Valley,E12000001,North East,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000007,West Midlands,E12000005,West Midlands,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000008,Cambridgeshire and Peterborough,E12000006,East of England,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000009,West of England,E12000009,South West,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000010,North East,E12000001,North East,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E47000011,North of Tyne,E12000001,North East,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
diff --git a/datastore/tests/files/geolookups/ctry_all_codes.csv b/datastore/tests/files/geolookups/ctry_all_codes.csv
@@ -0,0 +1,5 @@
+CTRYCD,CTRYNM,EWCD,EWNM,GBCD,GBNM,UKCD,UKNM
+E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+N92000002,Northern Ireland,,,,,K02000001,United Kingdom
+S92000003,Scotland,,,K03000001,Great Britain,K02000001,United Kingdom
+W92000004,Wales,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
diff --git a/datastore/tests/files/geolookups/rgn_all_codes.csv b/datastore/tests/files/geolookups/rgn_all_codes.csv
@@ -0,0 +1,10 @@
+RGNCD,RGNNM,CTRYCD,CTRYNM,EWCD,EWNM,GBCD,GBNM,UKCD,UKNM
+E12000001,North East,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000002,North West,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000003,Yorkshire and The Humber,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000004,East Midlands,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000005,West Midlands,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000009,South West,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000006,East of England,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000008,South East,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom
+E12000007,London,E92000001,England,K04000001,England and Wales,K03000001,Great Britain,K02000001,United Kingdom