Permalink
Browse files

Merge pull request #90 from sbrunner/sqs-filtred

Always delete tiles form SQS
  • Loading branch information...
2 parents 761f210 + 296eaaf commit 8c7a049839e732a1273f37b4470f652561a1db77 @sbrunner sbrunner committed Mar 24, 2013
@@ -784,10 +784,11 @@ class HashDropper(object):
The ``store`` is used to delete the empty tiles.
"""
- def __init__(self, size, sha1code, store=None):
+ def __init__(self, size, sha1code, store=None, queue_store=None):
self.size = size
self.sha1code = sha1code
self.store = store
+ self.queue_store = queue_store
def __call__(self, tile):
if len(tile.data) != self.size or \
@@ -801,6 +802,10 @@ def __call__(self, tile):
else:
self.store.delete_one(tile)
logger.info("The tile %s is dropped" % str(tile.tilecoord))
+ if self.queue_store is not None:
+ self.queue_store.delete_one(tile)
+ if hasattr(tile, 'metatile'):
+ tile.metatile.elapsed_togenerate -= 1
return None
@@ -107,7 +107,10 @@ def _gene(options, gene, layer):
# Discard tiles with certain content
if meta and 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
- gene.imap(HashDropper(empty_tile['size'], empty_tile['hash'], store=cache_tilestore))
+ gene.imap(HashDropper(
+ empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
+ queue_store=sqs_tilestore
+ ))
def add_elapsed_togenerate(metatile):
if metatile is not None:
@@ -119,17 +122,16 @@ def add_elapsed_togenerate(metatile):
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
- if options.role != 'hash':
- # Only keep tiles that intersect geometry
- gene.add_geom_filter(sqs_tilestore)
-
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
- gene.imap(HashDropper(empty_tile['size'], empty_tile['hash'], store=cache_tilestore))
+ gene.imap(HashDropper(
+ empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
+ queue_store=sqs_tilestore if not meta else None
+ ))
if options.role in ('local', 'slave'):
if options.test > 0 or options.verbose:
@@ -1351,6 +1351,8 @@ def test_mapcache(self):
connection: user=postgres password=postgres dbname=tests host=localhost
cost: *id001
dimensions: *id002
+ empty_metatile_detection: {hash: 01062bb3b25dcead792d7824f9a7045f0dd92992, size: 20743}
+ empty_tile_detection: {hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8, size: 334}
extension: png
grid: swissgrid_5
grid_ref: *id003
@@ -1374,9 +1376,9 @@ def test_mapcache(self):
grid: swissgrid_5
grid_ref: *id003
mapfile: mapfile/test.mapnik
- meta: true
- meta_buffer: 0
- meta_size: 8
+ meta: false
+ meta_buffer: 128
+ meta_size: 1
mime_type: image/png
name: mapnik
px_buffer: false
@@ -1400,7 +1402,7 @@ def test_mapcache(self):
polygon: [name]
mapfile: mapfile/test.mapnik
meta: false
- meta_buffer: 0
+ meta_buffer: 128
meta_size: 1
mime_type: application/utfgrid
name: mapnik_grid
@@ -1518,6 +1520,8 @@ def test_mapcache(self):
connection: user=postgres password=postgres dbname=tests host=localhost
cost: *id001
dimensions: *id002
+ empty_metatile_detection: {hash: 01062bb3b25dcead792d7824f9a7045f0dd92992, size: 20743}
+ empty_tile_detection: {hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8, size: 334}
extension: png
grid: swissgrid_5
grid_ref: *id003
@@ -1536,6 +1540,8 @@ def test_mapcache(self):
connection: user=postgres password=postgres dbname=tests host=localhost
cost: *id001
dimensions: *id002
+ empty_metatile_detection: {hash: 01062bb3b25dcead792d7824f9a7045f0dd92992, size: 20743}
+ empty_tile_detection: {hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8, size: 334}
extension: png
grid: swissgrid_01
grid_ref: *id004
@@ -36,11 +36,7 @@ def test_hash_mapnik(self):
self.assert_cmd_equals(
'./buildout/bin/generate_tiles --get-hash 4/0/0 -c tilegeneration/test.yaml -l mapnik',
generate.main,
- """Tile: 4/0/0:+8/+8
- empty_metatile_detection:
- size: 16375
- hash: b1087dd40c5d54e70c1824355a7dec802224c7f5
-Tile: 4/0/0
+ """Tile: 4/0/0
empty_tile_detection:
size: 334
hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8""")
@@ -60,7 +56,7 @@ def test_test_all(self):
generate.main,
"/tmp/tiles/",
'1.0.0/%s/default/2012/swissgrid_5/%i/%i/%i.png', [
- ('line', 0, 0, 0), ('polygon', 0, 5, 4)
+ ('line', 0, 7, 4), ('polygon', 0, 5, 4)
]
)
@@ -119,30 +119,47 @@ layers:
line:
layers: line
sql: the_geom AS geom FROM tests.line
+ empty_metatile_detection:
+ size: 20743
+ hash: 01062bb3b25dcead792d7824f9a7045f0dd92992
+ empty_tile_detection:
+ size: 334
+ hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8
polygon:
layers: polygon
meta: off
sql: the_geom AS geom FROM tests.polygon
+ empty_metatile_detection:
+ size: 20743
+ hash: 01062bb3b25dcead792d7824f9a7045f0dd92992
+ empty_tile_detection:
+ size: 334
+ hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8
polygon2:
layers: polygon
grid: swissgrid_01
sql: the_geom AS geom FROM tests.polygon
+ empty_metatile_detection:
+ size: 20743
+ hash: 01062bb3b25dcead792d7824f9a7045f0dd92992
+ empty_tile_detection:
+ size: 334
+ hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8
all:
layers: point,line,polygon
meta: off
bbox: [550000, 170000, 560000, 180000]
mapnik:
type: mapnik
mapfile: mapfile/test.mapnik
- meta_buffer: 0
+ meta: off
data_buffer: 128
output_format: png
sql: the_geom AS geom FROM tests.polygon
mapnik_grid:
type: mapnik
mapfile: mapfile/test.mapnik
meta: off
- meta_buffer: 0
data_buffer: 128
output_format: grid
mime_type: application/utfgrid

0 comments on commit 8c7a049

Please sign in to comment.