Skip to content

Commit

Permalink
Merge d5651e7 into b6cc82d
Browse files Browse the repository at this point in the history
  • Loading branch information
@sbrunner
sbrunner committed Jun 2, 2013
2 parents b6cc82d + d5651e7 commit 5e98b50
Show file tree
Hide file tree
Showing 8 changed files with 369 additions and 49 deletions.
55 changes: 51 additions & 4 deletions tilecloud_chain/__init__.py
View file
Expand Up @@ -655,6 +655,8 @@ def safe_get(tilestream):
yield metatile
self.tilestream = safe_get(self.tilestream)

error_file = None

def add_error_filters(self, logger):
self.imap(LogErrors(
logger, logging.ERROR,
Expand All @@ -663,6 +665,16 @@ def add_error_filters(self, logger):
if 'maxconsecutive_errors' in self.config['generation']:
self.tilestream = imap(MaximumConsecutiveErrors(
self.config['generation']['maxconsecutive_errors']), self.tilestream)
if 'error_file' in self.config['generation']:
if self.error_file is None:
self.error_file = open(self.config['generation']['error_file'], 'a')
self.error_file.write('# Start import at %s\n' % datetime.now().strftime('%d-%m-%Y %H:%M:%S'))

def do(tile):
if tile and tile.error:
self.error_file.write('%s # %s\n' % (tile.tilecoord, tile.error.replace('\n', ' ')))
return tile
self.imap(do)
self.ifilter(DropErrors())

def init_tilecoords(self, options):
Expand Down Expand Up @@ -852,7 +864,7 @@ def safe_filter(tile):
self.tilestream = ifilter(safe_filter, self.tilestream)


class HashDropper(object):
class HashDropper:
"""
Create a filter to remove the tiles data where they have
the specified size and hash.
Expand Down Expand Up @@ -887,7 +899,7 @@ def __call__(self, tile):
return None


class HashLogger(object):
class HashLogger:
"""
Log the tile size and hash.
"""
Expand Down Expand Up @@ -915,7 +927,7 @@ def __call__(self, tile):
return tile


class IntersectGeometryFilter(object):
class IntersectGeometryFilter:

def __init__(self, grid, geom=None, queue_store=None, px_buffer=0):
self.grid = grid
Expand All @@ -942,7 +954,7 @@ def bbox_polygon(self, bbox):
))


class DropEmpty(object):
class DropEmpty:
"""
Create a filter for dropping all tiles with errors.
"""
Expand All @@ -952,3 +964,38 @@ def __call__(self, tile):
return None # pragma: no cover
else:
return tile


def parse_tilecoord(string_representation):
parts = string_representation.split(':')
coords = [int(v) for v in parts[0].split('/')]
if len(coords) != 3: # pragma: no cover
raise ValueError("Wrong number of coordinates")
z, x, y = coords
if len(parts) == 1:
tilecoord = TileCoord(z, x, y)
elif len(parts) == 2:
meta = parts[1].split('/')
if len(meta) != 2: # pragma: no cover
raise ValueError("No one '/' in meta coordinates")
tilecoord = TileCoord(z, x, y, int(meta[0]))
else: # pragma: no cover
raise ValueError("More than on ':' in the tilecoord")
return tilecoord


class TilesFileStore:
def __init__(self, tiles_file):
self.tiles_file = open(tiles_file)

def list(self):
while True:
line = self.tiles_file.readline()
if not line:
return
line = line.split('#')[0].strip()
if line != '':
try:
yield Tile(parse_tilecoord(line))
except ValueError as e: # pragma: no cover
logger.error("A tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (line, e))
53 changes: 29 additions & 24 deletions tilecloud_chain/generate.py
View file
Expand Up @@ -13,7 +13,8 @@
from tilecloud.layout.wms import WMSTileLayout
from tilecloud.filter.logger import Logger

from tilecloud_chain import TileGeneration, HashDropper, HashLogger, DropEmpty, add_comon_options
from tilecloud_chain import TileGeneration, HashDropper, HashLogger, DropEmpty, TilesFileStore, \
add_comon_options, parse_tilecoord

logger = logging.getLogger(__name__)

Expand All @@ -25,17 +26,8 @@ def _gene(options, gene, layer):
gene.set_layer(layer, options)

if options.get_bbox:
if not gene.layer:
exit("A layer is required with --get-bbox option") # pragma: no cover
try:
parts = options.get_bbox.split(':')
z, x, y = (int(v) for v in parts[0].split('/'))
if len(parts) == 1:
tilecoord = TileCoord(z, x, y)
elif len(parts) == 2:
tilecoord = TileCoord(z, x, y, int(parts[1].split('/')[0]))
else: # pragma: no cover
exit("Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n" % options.get_bbox)
tilecoord = parse_tilecoord(options.get_bbox)
print \
"Tile bounds: [%i,%i,%i,%i]" % \
gene.layer['grid_ref']['obj'].extent(tilecoord)
Expand All @@ -62,7 +54,10 @@ def _gene(options, gene, layer):
exit('unknown cache type: ' + cache['type']) # pragma: no cover

meta = gene.layer['meta']
if options.role in ('local', 'master'):
if options.tiles_file:
gene.set_store(TilesFileStore(options.tiles_file))

elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords(options)
gene.add_geom_filter()
Expand All @@ -72,8 +67,6 @@ def _gene(options, gene, layer):
gene.set_store(sqs_tilestore) # pragma: no cover

elif options.role == 'hash':
if not gene.layer:
exit("A layer is required with --get-hash option") # pragma: no cover
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if meta:
Expand Down Expand Up @@ -232,7 +225,6 @@ def daemonize(): # pragma: no cover


def main():

parser = OptionParser('Used to generate the tiles')
add_comon_options(parser)
parser.add_option(
Expand All @@ -252,6 +244,10 @@ def main():
'--get-bbox', metavar="TILE",
help='get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_option(
'--tiles-file', metavar="FILE",
help='Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)

(options, args) = parser.parse_args()

Expand All @@ -268,12 +264,21 @@ def main():
if options.cache is None:
options.cache = gene.config['generation']['default_cache']

if (options.layer):
_gene(options, gene, options.layer)
elif options.get_bbox:
exit("With --get-bbox option we needs to specify a layer") # pragma: no cover
elif options.get_hash:
exit("With --get-hash option we needs to specify a layer") # pragma: no cover
else:
for layer in gene.config['generation']['default_layers']:
_gene(options, gene, layer)
if options.tiles_file and options.role not in ['local', 'master']: # pragma: no cover
exit("The --tiles-file option worky only with role local or master")

try:
if (options.layer):
_gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles_file: # pragma: no cover
exit("With --tiles-file option we needs to specify a layer")
else:
for layer in gene.config['generation']['default_layers']:
_gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
12 changes: 12 additions & 0 deletions tilecloud_chain/tests/__init__.py
View file
Expand Up @@ -72,6 +72,18 @@ def assert_main_equals(self, cmd, main_func, expected, **kargs):
f = open(expect[0], 'r')
self.assert_result_equals(f.read(), expect[1], **kargs)

def assert_main_except_equals(self, cmd, main_func, expected, **kargs):
sys.argv = cmd.split(' ')
try:
main_func()
assert("exit() not called.") # pragma: no cover
except:
pass
if expected:
for expect in expected:
f = open(expect[0], 'r')
self.assert_result_equals(f.read(), expect[1], **kargs)

def assert_yaml_equals(self, content, value):
import yaml
content = yaml.dump(yaml.load(content), width=120)
Expand Down
21 changes: 11 additions & 10 deletions tilecloud_chain/tests/test_controller.py
View file
Expand Up @@ -25,7 +25,7 @@ def tearDownClass(self):
@attr(general=True)
def test_capabilities(self):
self.assert_main_equals(
cmd='./buildout/bin/generate_controller --capabilities -c tilegeneration/test.yaml',
cmd='./buildout/bin/generate_controller --capabilities -c tilegeneration/test_fix.yaml',
main_func=controller.main,
expected=[[
'/tmp/tiles/1.0.0/WMTSCapabilities.xml',
Expand Down Expand Up @@ -935,7 +935,7 @@ def test_capabilities(self):
@attr(general=True)
def test_multi_host_capabilities(self):
self.assert_main_equals(
cmd='./buildout/bin/generate_controller --capabilities -c tilegeneration/test.yaml '
cmd='./buildout/bin/generate_controller --capabilities -c tilegeneration/test_fix.yaml '
'--destination-cache multi_host',
main_func=controller.main,
expected=[['/tmp/tiles/1.0.0/WMTSCapabilities.xml', self.MULTIHOST_CAPABILITIES]])
Expand All @@ -944,7 +944,7 @@ def test_multi_host_capabilities(self):
@attr(general=True)
def test_multi_url_capabilities(self):
self.assert_main_equals(
cmd='./buildout/bin/generate_controller --capabilities -c tilegeneration/test.yaml '
cmd='./buildout/bin/generate_controller --capabilities -c tilegeneration/test_fix.yaml '
'--destination-cache multi_url',
main_func=controller.main,
expected=[['/tmp/tiles/1.0.0/WMTSCapabilities.xml', self.MULTIHOST_CAPABILITIES]])
Expand All @@ -953,7 +953,7 @@ def test_multi_url_capabilities(self):
@attr(general=True)
def test_mapcache(self):
self.assert_main_equals(
cmd='./buildout/bin/generate_controller --mapcache -c tilegeneration/test.yaml',
cmd='./buildout/bin/generate_controller --mapcache -c tilegeneration/test_fix.yaml',
main_func=controller.main,
expected=[['mapcache.xml', u"""<?xml version="1.0" encoding="UTF-8"?>
<mapcache>
Expand Down Expand Up @@ -1240,7 +1240,7 @@ def test_mapcache(self):
@attr(general=True)
def test_apache(self):
self.assert_main_equals(
cmd='./buildout/bin/generate_controller --apache -c tilegeneration/test.yaml',
cmd='./buildout/bin/generate_controller --apache -c tilegeneration/test_fix.yaml',
main_func=controller.main,
expected=[['tiles.conf.in', u"""<Location /${vars:instanceid}/tiles>
ExpiresActive on
Expand Down Expand Up @@ -1298,6 +1298,7 @@ def test_apache(self):
disable_geodata: false
disable_tilesgen: false
ec2_host_type: m1.medium
error_file: error.list
geodata_folder: tilecloud_chain/
maxconsecutive_errors: 2
number_process: 1
Expand Down Expand Up @@ -1592,14 +1593,14 @@ def test_apache(self):
@attr(general=True)
def test_config(self):
self.assert_cmd_yaml_equals(
cmd='./buildout/bin/generate_controller --dump-config -c tilegeneration/test.yaml',
cmd='./buildout/bin/generate_controller --dump-config -c tilegeneration/test_fix.yaml',
main_func=controller.main, result=self.CONFIG)

@attr(config_layer=True)
@attr(general=True)
def test_config_layer(self):
self.assert_cmd_yaml_equals(
cmd='./buildout/bin/generate_controller --dump-config -l line -c tilegeneration/test.yaml',
cmd='./buildout/bin/generate_controller --dump-config -l line -c tilegeneration/test_fix.yaml',
main_func=controller.main, result=self.CONFIG)

@attr(openlayers=True)
Expand Down Expand Up @@ -1749,7 +1750,7 @@ def test_openlayers(self):
}
});"""
self.assert_main_equals(
cmd='./buildout/bin/generate_controller --ol -c tilegeneration/test.yaml',
cmd='./buildout/bin/generate_controller --ol -c tilegeneration/test_fix.yaml',
main_func=controller.main,
expected=[
['/tmp/tiles/index.html', html],
Expand All @@ -1758,7 +1759,7 @@ def test_openlayers(self):
)

self.assert_main_equals(
cmd='./buildout/bin/generate_controller --ol -c tilegeneration/test.yaml --cache multi_host',
cmd='./buildout/bin/generate_controller --ol -c tilegeneration/test_fix.yaml --cache multi_host',
main_func=controller.main,
expected=[
['/tmp/tiles/index.html', html],
Expand All @@ -1767,7 +1768,7 @@ def test_openlayers(self):
)

self.assert_main_equals(
cmd='./buildout/bin/generate_controller --ol -c tilegeneration/test.yaml --cache multi_url',
cmd='./buildout/bin/generate_controller --ol -c tilegeneration/test_fix.yaml --cache multi_url',
main_func=controller.main,
expected=[
['/tmp/tiles/index.html', html],
Expand Down

0 comments on commit 5e98b50

Please sign in to comment.