Merge 590867a into 1806fd3

README.md

@@ -47,7 +47,7 @@ $ manage.py load_datagetter_data ../path/to/data/dir/from/datagetter/
 A number of the sources for additional_data have their own local caches these can be loaded via:
 
 ```
-$ manage.py load_code_names
+$ manage.py load_geocode_names
 $ manage.py load_geolookups
 $ manage.py load_nspl
 $ manage.py load_org_data
@@ -83,13 +83,13 @@ Whilst leaving the up command running, you should use `docker-compose run` with
 eg; instead of running:
 
 ```
-$ manage.py load_code_names
+$ manage.py load_geocode_names
 ```
 
 Run:
 
 ```
-$ docker-compose -f docker-compose.dev.yml run datastore-web python datastore/manage.py load_code_names
+$ docker-compose -f docker-compose.dev.yml run datastore-web python datastore/manage.py load_geocode_names
 ```
 
 ## Getting database CLI

datastore/additional_data/generator.py

@@ -6,6 +6,7 @@
 from additional_data.sources.additional_data_recipient_location import (
     AdditionalDataRecipientLocation,
 )
+from additional_data.sources.codelist_code import CodeListSource
 
 
 class AdditionalDataGenerator(object):
@@ -18,6 +19,7 @@ def __init__(self):
         self.geo_lookup = GeoLookupSource()
         self.tsg_org_types = TSGOrgTypesSource()
         self.additional_data_recipient_location = AdditionalDataRecipientLocation()
+        self.code_lists = CodeListSource()
         # Initialise Other Sources here
 
     def create(self, grant):
@@ -35,5 +37,6 @@ def create(self, grant):
         self.additional_data_recipient_location.update_additional_data(
             grant, additional_data
         )
+        self.code_lists.update_additional_data(grant, additional_data)
 
         return additional_data

datastore/additional_data/management/commands/load_codelist_codes.py

@@ -0,0 +1,11 @@
+from django.core.management.base import BaseCommand
+
+from additional_data.sources.codelist_code import CodeListSource
+
+
+class Command(BaseCommand):
+    help = "Imports 360Giving standard codelist data"
+
+    def handle(self, *args, **options):
+        source = CodeListSource()
+        source.import_codelists()

datastore/additional_data/management/commands/load_code_names.py → datastore/additional_data/management/commands/load_geocode_names.py

@@ -1,11 +1,11 @@
 from django.core.management.base import BaseCommand
 
-from additional_data.sources.code_names import CodeNamesSource
+from additional_data.sources.code_names import GeoCodeNamesSource
 
 
 class Command(BaseCommand):
     help = "Imports location code names"
 
     def handle(self, *args, **options):
-        source = CodeNamesSource()
+        source = GeoCodeNamesSource()
         source.import_code_names()

datastore/additional_data/migrations/0008_auto_20230123_1727.py

@@ -0,0 +1,47 @@
+# Generated by Django 3.2.16 on 2023-01-23 17:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("additional_data", "0007_auto_20200918_0954"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="GeoCodeName",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("code", models.CharField(db_index=True, max_length=9)),
+                ("data", models.JSONField()),
+            ],
+        ),
+        migrations.DeleteModel(
+            name="CodeName",
+        ),
+        migrations.AlterField(
+            model_name="geolookup",
+            name="data",
+            field=models.JSONField(),
+        ),
+        migrations.AlterField(
+            model_name="nspl",
+            name="data",
+            field=models.JSONField(),
+        ),
+        migrations.AlterField(
+            model_name="orginfocache",
+            name="data",
+            field=models.JSONField(),
+        ),
+    ]

datastore/additional_data/migrations/0009_codelistcode.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.16 on 2023-01-24 13:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("additional_data", "0008_auto_20230123_1727"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CodelistCode",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "list_name",
+                    models.CharField(
+                        help_text="The name of the CodeList the code belongs to",
+                        max_length=200,
+                    ),
+                ),
+                ("code", models.CharField(help_text="The code", max_length=200)),
+                (
+                    "title",
+                    models.CharField(help_text="The title of the code", max_length=200),
+                ),
+                (
+                    "description",
+                    models.TextField(help_text="The long description of the code"),
+                ),
+            ],
+            options={
+                "unique_together": {("list_name", "code")},
+            },
+        ),
+    ]

datastore/additional_data/models.py

@@ -100,7 +100,7 @@ class NSPL(models.Model):
     data = JSONField()
 
 
-class CodeName(models.Model):
+class GeoCodeName(models.Model):
     code = models.CharField(max_length=9, db_index=True)
     data = JSONField()
 
@@ -156,3 +156,17 @@ class Meta:
 
     def __str__(self):
         return self.tsg_org_type
+
+
+class CodelistCode(models.Model):
+    """360Giving standard code lists codes and titles"""
+
+    list_name = models.CharField(
+        max_length=200, help_text="The name of the codelist the code belongs to"
+    )
+    code = models.CharField(max_length=200, help_text="The code")
+    title = models.CharField(max_length=200, help_text="The title of the code")
+    description = models.TextField(help_text="The long description of the code")
+
+    class Meta:
+        unique_together = ("list_name", "code")

datastore/additional_data/sources/codelist_code.py

@@ -0,0 +1,90 @@
+import csv
+import requests
+
+from additional_data.models import CodelistCode
+
+code_lists_urls = [
+    "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/master/codelists/grantToIndividualsPurpose.csv",
+    "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/master/codelists/grantToIndividualsReason.csv",
+    "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/master/codelists/regrantType.csv",
+    # These lists aren't yet ready for use in the datastore
+    # https://github.com/ThreeSixtyGiving/standard/issues/348
+    # https://github.com/ThreeSixtyGiving/standard/issues/349
+    # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/master/codelists/countryCode.csv",
+    # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/master/codelists/currency.csv",
+    # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/master/codelists/geoCodeType.csv",
+]
+
+
+class CodeListSource(object):
+    """Looks up codes from 360Giving codelists and gets the title value of the code
+    responsible for field: codeListLookup
+    """
+
+    def import_codelists(self):
+        CodelistCode.objects.all().delete()
+
+        for code_list_url in code_lists_urls:
+            # list name = last item in split -4 to remove extension .csv
+            list_name = code_list_url.split("/")[-1:][0][:-4]
+            with requests.get(code_list_url, stream=True) as r:
+                r.raise_for_status()
+                file_data = csv.DictReader(
+                    r.iter_lines(decode_unicode=True), delimiter=","
+                )
+                for value in file_data:
+                    CodelistCode.objects.create(
+                        code=value["Code"],
+                        title=value["Title"],
+                        description=value["Description"],
+                        list_name=list_name,
+                    )
+
+    def update_additional_data(self, grant, additional_data):
+        # check All the fields in the grant data that use codelists and make additional data field versions of them
+
+        primaryGrantReason = ""
+        secondaryGrantReason = ""
+        grantPurpose = ""
+        regrantType = ""
+
+        try:
+            code = grant["toIndividualDetails"]["primaryGrantReason"]
+            primaryGrantReason = CodelistCode.objects.get(
+                code=code, list_name="grantToIndividualsReason"
+            ).title
+        except (KeyError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["toIndividualDetails"]["secondaryGrantReason"]
+            secondaryGrantReason = CodelistCode.objects.get(
+                code=code, list_name="grantToIndividualsReason"
+            ).title
+        except (KeyError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["toIndividualDetails"]["grantPurpose"]
+            grantPurpose = CodelistCode.objects.get(
+                code=code, list_name="grantToIndividualsPurpose"
+            ).title
+        except (KeyError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["regrantType"]
+            regrantType = CodelistCode.objects.get(
+                code=code, list_name="regrantType"
+            ).title
+        except (KeyError, CodelistCode.DoesNotExist):
+            pass
+
+        additional_data["codeListLookup"] = {
+            "toIndividualDetails": {
+                "primaryGrantReason": primaryGrantReason,
+                "secondaryGrantReason": secondaryGrantReason,
+                "grantPurpose": grantPurpose,
+            },
+            "regrantType": regrantType,
+        }

datastore/additional_data/sources/code_names.py → datastore/additional_data/sources/geocode_names.py

@@ -5,14 +5,14 @@
 
 import requests
 
-from additional_data.models import CodeName
+from additional_data.models import GeoCodeName
 
 # based on 'import_chd' function in
 # https://github.com/drkane/find-that-postcode/blob/master/findthatpostcode/commands/codes.py
 
 
-class CodeNamesSource(object):
-    """Uses CHD (Change history data) at https://geoportal.statistics.gov.uk/ to obtain information about code names."""
+class GeoCodeNamesSource(object):
+    """Uses CHD (Change history data) at https://geoportal.statistics.gov.uk/ to obtain information about geo code names."""
 
     CHD_URL = "https://www.arcgis.com/sharing/rest/content/items/56b8f6d2d26646cb9d21fadca2f09452/data"
 
@@ -128,9 +128,9 @@ def save_data(self, areas):
         bulk_save = []
 
         for code, data in areas.items():
-            bulk_save.append(CodeName(code=code, data=data))
+            bulk_save.append(GeoCodeName(code=code, data=data))
 
-        CodeName.objects.bulk_create(bulk_save)
+        GeoCodeName.objects.bulk_create(bulk_save)
 
     def import_code_names(self, url=CHD_URL):
         """
@@ -147,7 +147,7 @@ def import_code_names(self, url=CHD_URL):
         zip_file = self.get_zipfile(url)
         areas = self.get_areas(zip_file)
 
-        if CodeName.objects.exists():
-            CodeName.objects.all().delete()
+        if GeoCodeName.objects.exists():
+            GeoCodeName.objects.all().delete()
 
         self.save_data(areas)

datastore/additional_data/sources/nspl.py

@@ -6,7 +6,7 @@
 
 import requests
 
-from additional_data.models import NSPL, CodeName
+from additional_data.models import NSPL, GeoCodeName
 
 # based on https://github.com/drkane/find-that-postcode/blob/master/findthatpostcode/commands/postcodes.py
 
@@ -161,9 +161,9 @@ def update_location_data_code_names(self, location_data):
                     continue
             except KeyError:
                 try:
-                    code_name_obj = CodeName.objects.get(code=field_value)
+                    code_name_obj = GeoCodeName.objects.get(code=field_value)
                     self._code_name_cache[field_value] = code_name_obj
-                except CodeName.DoesNotExist:
+                except GeoCodeName.DoesNotExist:
                     self._code_name_cache[field_value] = None
                     continue