Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Always delete tiles form SQS #90

Merged
merged 1 commit into from

1 participant

@sbrunner
Owner

fix #89

@sbrunner sbrunner merged commit 8c7a049 into master
@sbrunner sbrunner deleted the sqs-filtred branch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 24, 2013
  1. Always delete tiles form SQS

    authored committed
This page is out of date. Refresh to see the latest.
View
7 tilecloud_chain/__init__.py
@@ -784,10 +784,11 @@ class HashDropper(object):
The ``store`` is used to delete the empty tiles.
"""
- def __init__(self, size, sha1code, store=None):
+ def __init__(self, size, sha1code, store=None, queue_store=None):
self.size = size
self.sha1code = sha1code
self.store = store
+ self.queue_store = queue_store
def __call__(self, tile):
if len(tile.data) != self.size or \
@@ -801,6 +802,10 @@ def __call__(self, tile):
else:
self.store.delete_one(tile)
logger.info("The tile %s is dropped" % str(tile.tilecoord))
+ if self.queue_store is not None:
+ self.queue_store.delete_one(tile)
+ if hasattr(tile, 'metatile'):
+ tile.metatile.elapsed_togenerate -= 1
return None
View
14 tilecloud_chain/generate.py
@@ -107,7 +107,10 @@ def _gene(options, gene, layer):
# Discard tiles with certain content
if meta and 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
- gene.imap(HashDropper(empty_tile['size'], empty_tile['hash'], store=cache_tilestore))
+ gene.imap(HashDropper(
+ empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
+ queue_store=sqs_tilestore
+ ))
def add_elapsed_togenerate(metatile):
if metatile is not None:
@@ -119,17 +122,16 @@ def add_elapsed_togenerate(metatile):
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
- if options.role != 'hash':
- # Only keep tiles that intersect geometry
- gene.add_geom_filter(sqs_tilestore)
-
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
- gene.imap(HashDropper(empty_tile['size'], empty_tile['hash'], store=cache_tilestore))
+ gene.imap(HashDropper(
+ empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
+ queue_store=sqs_tilestore if not meta else None
+ ))
if options.role in ('local', 'slave'):
if options.test > 0 or options.verbose:
View
14 tilecloud_chain/tests/test_controller.py
@@ -1351,6 +1351,8 @@ def test_mapcache(self):
connection: user=postgres password=postgres dbname=tests host=localhost
cost: *id001
dimensions: *id002
+ empty_metatile_detection: {hash: 01062bb3b25dcead792d7824f9a7045f0dd92992, size: 20743}
+ empty_tile_detection: {hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8, size: 334}
extension: png
grid: swissgrid_5
grid_ref: *id003
@@ -1374,9 +1376,9 @@ def test_mapcache(self):
grid: swissgrid_5
grid_ref: *id003
mapfile: mapfile/test.mapnik
- meta: true
- meta_buffer: 0
- meta_size: 8
+ meta: false
+ meta_buffer: 128
+ meta_size: 1
mime_type: image/png
name: mapnik
px_buffer: false
@@ -1400,7 +1402,7 @@ def test_mapcache(self):
polygon: [name]
mapfile: mapfile/test.mapnik
meta: false
- meta_buffer: 0
+ meta_buffer: 128
meta_size: 1
mime_type: application/utfgrid
name: mapnik_grid
@@ -1518,6 +1520,8 @@ def test_mapcache(self):
connection: user=postgres password=postgres dbname=tests host=localhost
cost: *id001
dimensions: *id002
+ empty_metatile_detection: {hash: 01062bb3b25dcead792d7824f9a7045f0dd92992, size: 20743}
+ empty_tile_detection: {hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8, size: 334}
extension: png
grid: swissgrid_5
grid_ref: *id003
@@ -1536,6 +1540,8 @@ def test_mapcache(self):
connection: user=postgres password=postgres dbname=tests host=localhost
cost: *id001
dimensions: *id002
+ empty_metatile_detection: {hash: 01062bb3b25dcead792d7824f9a7045f0dd92992, size: 20743}
+ empty_tile_detection: {hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8, size: 334}
extension: png
grid: swissgrid_01
grid_ref: *id004
View
8 tilecloud_chain/tests/test_generate.py
@@ -36,11 +36,7 @@ def test_hash_mapnik(self):
self.assert_cmd_equals(
'./buildout/bin/generate_tiles --get-hash 4/0/0 -c tilegeneration/test.yaml -l mapnik',
generate.main,
- """Tile: 4/0/0:+8/+8
- empty_metatile_detection:
- size: 16375
- hash: b1087dd40c5d54e70c1824355a7dec802224c7f5
-Tile: 4/0/0
+ """Tile: 4/0/0
empty_tile_detection:
size: 334
hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8""")
@@ -60,7 +56,7 @@ def test_test_all(self):
generate.main,
"/tmp/tiles/",
'1.0.0/%s/default/2012/swissgrid_5/%i/%i/%i.png', [
- ('line', 0, 0, 0), ('polygon', 0, 5, 4)
+ ('line', 0, 7, 4), ('polygon', 0, 5, 4)
]
)
View
21 tilecloud_chain/tests/tilegeneration/test.yaml
@@ -119,14 +119,32 @@ layers:
line:
layers: line
sql: the_geom AS geom FROM tests.line
+ empty_metatile_detection:
+ size: 20743
+ hash: 01062bb3b25dcead792d7824f9a7045f0dd92992
+ empty_tile_detection:
+ size: 334
+ hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8
polygon:
layers: polygon
meta: off
sql: the_geom AS geom FROM tests.polygon
+ empty_metatile_detection:
+ size: 20743
+ hash: 01062bb3b25dcead792d7824f9a7045f0dd92992
+ empty_tile_detection:
+ size: 334
+ hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8
polygon2:
layers: polygon
grid: swissgrid_01
sql: the_geom AS geom FROM tests.polygon
+ empty_metatile_detection:
+ size: 20743
+ hash: 01062bb3b25dcead792d7824f9a7045f0dd92992
+ empty_tile_detection:
+ size: 334
+ hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8
all:
layers: point,line,polygon
meta: off
@@ -134,7 +152,7 @@ layers:
mapnik:
type: mapnik
mapfile: mapfile/test.mapnik
- meta_buffer: 0
+ meta: off
data_buffer: 128
output_format: png
sql: the_geom AS geom FROM tests.polygon
@@ -142,7 +160,6 @@ layers:
type: mapnik
mapfile: mapfile/test.mapnik
meta: off
- meta_buffer: 0
data_buffer: 128
output_format: grid
mime_type: application/utfgrid
Something went wrong with that request. Please try again.