Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

made quad tree generation much much faster

  • Loading branch information...
commit 0b4e99538fd4e005d1d49ee755411ada5c69b748 1 parent b996564
@trey0 trey0 authored
Showing with 54 additions and 18 deletions.
  1. +18 −9 geocamLayer/bin/importData.py
  2. +36 −9 geocamLayer/quadTree.py
View
27 geocamLayer/bin/importData.py
@@ -8,6 +8,8 @@
import sys
import itertools
+from django.db import transaction
+
from geocamLayer.noaaExampleParser import readNoaaWeatherStations
from geocamLayer.quadTree import QuadTree
from geocamLayer.models import Feature, QuadTreeCell
@@ -22,22 +24,29 @@ def importFeatures(tree, features, maxNumFeatures):
sys.stdout.flush()
+@transaction.commit_manually
+def cleanData():
+ print 'deleting old data'
+ # for loops are a work around for bulk delete problem in sqlite
+ # with objects.all().delete()
+ for f in Feature.objects.all():
+ f.delete()
+ for q in QuadTreeCell.objects.all():
+ q.delete()
+ transaction.commit()
+
+
def importData(opts):
if opts.clean:
- print 'deleting old data'
- # for loops are a work around for bulk delete problem in sqlite
- # with objects.all().delete()
- for f in Feature.objects.all():
- f.delete()
- for q in QuadTreeCell.objects.all():
- q.delete()
+ cleanData()
tree = QuadTree()
if opts.noaa:
sys.stdout.write('noaa import')
importFeatures(tree, readNoaaWeatherStations(opts.noaa), opts.maxNumFeatures)
- print
- print
+ tree.finish()
+ print
+ print
tree.debugStats()
View
45 geocamLayer/quadTree.py
@@ -4,8 +4,11 @@
# All Rights Reserved.
# __END_LICENSE__
+import sys
import math
+from django.db import transaction
+
from geocamLayer.models import Feature, QuadTreeCell
MAX_FEATURES_PER_CELL = 10
@@ -18,7 +21,20 @@
class QuadTree(object):
def __init__(self):
- self.root = QuadTreeCell.getCellAtIndex((0, 0, 0))
+ self.cells = {}
+ self.features = []
+ self.root = self.getCellAtIndex((0, 0, 0))
+
+ def getCellAtIndex(self, index):
+ if index in self.cells:
+ cell = self.cells[index]
+ else:
+ zoom, x, y = index
+ cell = self.cells.setdefault(index, QuadTreeCell(zoom=zoom, x=x, y=y))
+ return cell
+
+ def getCellAtLonLat(self, zoom, lonLat):
+ return self.getCellAtIndex(QuadTreeCell.getIndexAtLonLat(zoom, lonLat))
def addFeature(self, feature):
self.addFeatureToCell(feature, self.root)
@@ -28,28 +44,39 @@ def addFeatureToCell(self, feature, cell):
if cell.isLeaf:
feature.cell = cell
- feature.save()
+ self.features.append(feature)
+ if not hasattr(cell, 'features'):
+ cell.features = []
+ cell.features.append(feature)
if cell.count >= MAX_FEATURES_PER_CELL and cell.zoom < MAX_ZOOM - 1:
self.splitCell(cell)
else:
self.addFeatureToZoom(feature, cell.zoom + 1)
- cell.save()
-
def addFeatureToZoom(self, feature, zoom):
- cell = (QuadTreeCell.getCellAtLonLat
+ cell = (self.getCellAtLonLat
(zoom, (feature.lng, feature.lat)))
self.addFeatureToCell(feature, cell)
- def createCell(self, zoom, x, y):
- return QuadTreeCell.objects.create(zoom=zoom, x=x, y=y)
-
def splitCell(self, cell):
cell.isLeaf = False
- for feature in Feature.objects.filter(cell=cell):
+ for feature in cell.features:
self.addFeatureToZoom(feature, cell.zoom + 1)
+ @transaction.commit_manually
+ def finish(self):
+ for cell in self.cells.itervalues():
+ cell.save()
+ sys.stdout.write('c')
+ sys.stdout.flush()
+ transaction.commit()
+ for feature in self.features:
+ feature.save()
+ sys.stdout.write('f')
+ sys.stdout.flush()
+ transaction.commit()
+
def debugStats(self):
print '=== DEBUG STATS ==='
zoom = 0
Please sign in to comment.
Something went wrong with that request. Please try again.