Changes to the loadshapefiles routine that *drastically* improve the …

…runtime.

boundaryservice/management/commands/loadshapefiles.py

@@ -17,6 +17,7 @@
 DEFAULT_SHAPEFILES_DIR = getattr(settings, 'SHAPEFILES_DIR', 'data/shapefiles')
 GEOMETRY_COLUMN = 'shape'
 
+
 class Command(BaseCommand):
     help = 'Import boundaries described by shapefiles.'
     option_list = BaseCommand.option_list + (
@@ -86,6 +87,7 @@ def load_set(self, kind, config, options):
         layer = datasources[0][0]
 
         # Create BoundarySet
+        log.info("Creating BoundarySet: %s" % kind)
         bset = BoundarySet.objects.create(
             name=kind,
             singular=config['singular'],
@@ -95,9 +97,11 @@ def load_set(self, kind, config, options):
             last_updated=config['last_updated'],
             href=config['href'],
             notes=config['notes'],
-            count=len(layer),
-            metadata_fields=layer.fields)
-
+            count=0,
+            metadata_fields=layer.fields
+        )
+        log.info("Created with slug %s and id %s" % (bset.slug, bset.id))
+
         for datasource in datasources:
             log.info("Loading %s from %s" % (kind, datasource.name))
             # Assume only a single-layer in shapefile
@@ -145,6 +149,7 @@ def add_boundaries_for_layer(self, config, layer, bset, database):
         transformer = CoordTransform(layer_srs, db_srs)
 
         for feature in layer:
+            log.debug("Processing boundary %s" % feature)
             # Transform the geometry to the correct SRS
             geometry = self.polygon_to_multipolygon(feature.geom)
             geometry.transform(transformer)
@@ -194,7 +199,7 @@ def add_boundaries_for_layer(self, config, layer, bset, database):
                 shape=geometry.wkt,
                 simple_shape=simple_geometry.wkt,
                 centroid=geometry.geos.centroid)
-        
+
 def create_datasources(path):
     if path.endswith('.zip'):
         path = temp_shapefile_from_zip(path)
@@ -211,7 +216,7 @@ def create_datasources(path):
         if fn.endswith('.shp'):
             sources.append(DataSource(fn))
     return sources
-    
+
 def temp_shapefile_from_zip(zip_path):
     """
     Given a path to a ZIP file, unpack it into a temp dir and return the path
@@ -220,6 +225,7 @@ def temp_shapefile_from_zip(zip_path):
 
     If you want to cleanup later, you can derive the temp dir from this path.
     """
+    log.info("Creating temporary SHP file from %s" % zip_path)
     zf = ZipFile(zip_path)
     tempdir = mkdtemp()
     shape_path = None
@@ -242,4 +248,3 @@ def temp_shapefile_from_zip(zip_path):
         raise ValueError("No shapefile found in zip")
 
     return shape_path
-

boundaryservice/models.py

@@ -1,10 +1,9 @@
 import re
-
 from django.contrib.gis.db import models
-
 from boundaryservice.fields import ListField, JSONField
 from boundaryservice.utils import get_site_url_root
 
+
 class SluggedModel(models.Model):
     """
     Extend this class to get a slug field and slug generated from a model
@@ -36,24 +35,26 @@ def unique_slug(self):
                 slug_txt = str(self)
             else:
                 return
-            slug = slugify(slug_txt)
-
-            itemModel = self.__class__
-            # the following gets all existing slug values
-            allSlugs = set(sl.values()[0] for sl in itemModel.objects.values("slug"))
-            if slug in allSlugs:
-                counterFinder = re.compile(r'-\d+$')
-                counter = 2
-                slug = "%s-%i" % (slug, counter)
-                while slug in allSlugs:
-                    slug = re.sub(counterFinder,"-%i" % counter, slug)
-                    counter += 1
-
-            setattr(self,"slug",slug)
-
-    def fully_qualified_url(self):        
+            original_slug = slugify(slug_txt)
+            queryset = self.__class__._default_manager.all()
+            if not queryset.filter(slug=original_slug).count():
+                setattr(self, "slug", original_slug)
+            else:
+                slug = ''
+                next = 2
+                while not slug or queryset.filter(slug=slug).count():
+                    slug = original_slug
+                    end = '-%s' % next
+                    if len(slug) + len(end) > 256:
+                        slug = slug[:200-len(end)]
+                    slug = '%s%s' % (slug, end)
+                    next += 1
+                setattr(self, "slug", slug)
+
+    def fully_qualified_url(self):
         return get_site_url_root() + self.get_absolute_url()
 
+
 class BoundarySet(SluggedModel):
     """
     A set of related boundaries, such as all Wards or Neighborhoods.
@@ -88,6 +89,7 @@ def __unicode__(self):
         """
         return unicode(self.name)
 
+
 class Boundary(SluggedModel):
     """
     A boundary object, such as a Ward or Neighborhood.