Merge pull request #36 from groovecoder/refactor-load-scripts-28

fixes #28 - consolidate case-loading commands

bpz/management/commands/load_cases.py

@@ -0,0 +1,96 @@
+from datetime import datetime
+from json import loads
+from sys import stderr
+from time import mktime
+
+from django.core.management.base import BaseCommand, CommandError
+from django.contrib.gis.gdal import DataSource
+from django.contrib.gis.geos import GEOSGeometry
+from django.db import IntegrityError
+from parsedatetime import Calendar
+
+from bpz.models import Case
+
+'''
+Import BPZ case data
+'''
+
+
+def parse_datetime(value):
+    calendar = Calendar()
+    raw_date = loads('"%s"' % value)
+    timestamp = mktime(calendar.parseDateText(raw_date))
+    return datetime.fromtimestamp(timestamp).date()
+
+
+def parse_feature_raw_value(feature, field_name):
+    raw_value = feature[field_name].value
+    value = raw_value
+    if '\r\n' in raw_value:
+        value = raw_value.split('\r\n', 1)[0]
+        stderr.write('Invalid %s: "%s". Using first line: "%s"\n' %
+                     (field_name, repr(raw_value), value))
+    return loads('"%s"' % value)
+
+
+# http://stackoverflow.com/a/19357056/571420
+def update_or_create(command, model, filter_kwargs, update_kwargs):
+    if not model.objects.filter(**filter_kwargs).update(**update_kwargs):
+        kwargs = filter_kwargs.copy()
+        kwargs.update(update_kwargs)
+        try:
+            model.objects.create(**kwargs)
+            print('New case "%s"' % kwargs['case_id'])
+            command.new += 1
+        except IntegrityError:
+            if not (model.objects.filter(**filter_kwargs)
+                    .update(**update_kwargs)):
+                raise
+    else:
+        print('Updated case "%s"' % update_kwargs['case_id'])
+        command.updated += 1
+
+
+class Command(BaseCommand):
+    args = '<cases.json>'
+    help = 'Imports case GeoJSON'
+    new, updated = 0, 0
+
+    def handle(self, *args, **options):
+        if len(args) != 1:
+            raise CommandError('Pass one json file to import')
+
+        # Load data
+        datasource = DataSource(args[0], encoding='iso-8859-1')
+        layer = datasource[0]
+        for feature in layer:
+            # Get the common attributes' values
+            attr = {
+                'object_id': feature['OBJECTID'].value,
+                'case_id': loads('"%s"' % feature['Case_'].value),
+                'status': feature['Status'].value,
+                'link': loads('"%s"' % feature['Link'].value),
+            }
+            attr['location'] = parse_feature_raw_value(feature, 'Location')
+            attr['hearing_date'] = parse_datetime(feature['Date_'].value)
+            attr['geom'] = GEOSGeometry(feature.geom.wkt)
+
+            # Get the custom attribute values: case_type & domain
+            domain = Case.DOMAIN_TMAPC
+            case_type = None
+            if 'BOA' in attr['case_id']:
+                domain = Case.DOMAIN_BOA
+                case_type = Case.NAME_BOA
+            else:
+                case_type = parse_feature_raw_value(feature, 'Type')
+            attr['case_type'] = case_type
+
+            filter_kwargs = {'object_id': attr['object_id'],
+                             'domain': domain}
+            update_or_create(self, Case, filter_kwargs, attr)
+
+        self.stdout.write(
+            'Imported %d %s cases (%d new, %d updated)' %
+            (self.new + self.updated, Case.DOMAIN_CHOICES[domain],
+             self.new, self.updated)
+        )

bpz/models.py

@@ -13,11 +13,13 @@ class Case(models.Model):
     SHAPE_Leng - derive from geometry
     SHAPE_Area - derive from geometry
     '''
+    NAME_BOA = "Board of Adjustment"
+    NAME_TMAPC = "Tulsa Metropolitan Area Planning Commission"
     DOMAIN_BOA = 0
     DOMAIN_TMAPC = 1
     DOMAIN_CHOICES = (
-        (DOMAIN_BOA, 'Board of Adjustment'),
-        (DOMAIN_TMAPC, 'Tulsa Metropolitan Area Planning Commission'),
+        (DOMAIN_BOA, NAME_BOA),
+        (DOMAIN_TMAPC, NAME_TMAPC),
     )
 
     STATUS_CHOICES = [(x, x) for x in (
@@ -28,7 +30,7 @@ class Case(models.Model):
     )]
 
     CASE_TYPE_CHOICES = [(x, x) for x in (
-        "Board of Adjustment",
+        NAME_BOA,
         "Corridor Minor Amendment",
         "Corridor Plan",
         "Lot Split",
@@ -47,6 +49,7 @@ class Case(models.Model):
     location = models.CharField(max_length=50)
     link = models.URLField()
     hearing_date = models.DateField()
+    # TODO: remove choices constraint from case_type field
     case_type = models.CharField(max_length=30, choices=CASE_TYPE_CHOICES)
     geom = models.GeometryField(srid=4326)
     objects = models.GeoManager()

requirements.dev.txt

@@ -6,6 +6,8 @@ pep8==1.5.7
 pyflakes==0.8.1
 flake8==2.2.2
 
+mock==1.0.1
+
 # coverage - line and branch coverage
 coverage==3.7.1