From 5eabf32015cbc57e1694b30e34f50356d5c1eba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 10 Apr 2014 15:46:26 +0200 Subject: [PATCH 1/3] By default don't run the post process on dropped tiles --- README.rst | 3 ++- tilecloud_chain/__init__.py | 3 +++ tilecloud_chain/generate.py | 6 +++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index de8f0872..4ae384ac 100644 --- a/README.rst +++ b/README.rst @@ -320,7 +320,8 @@ Process ------- We can configure some tile commands to process the tiles. -They can be automatically be called in the tile generation it we set the property ``post_process`` in the layer configuration. +They can be automatically be called in the tile generation it we set the property +``post_process`` or ``pre_hash_post_process`` in the layer configuration. The process is a set of names processes, and each one has a list of commands declared like this: diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index 3b11b62f..d3f204fc 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -354,6 +354,9 @@ def __init__(self, config_file, options=None, layer_name=None): regex="^[a-zA-Z0-9_]+$" ) or error + error = self.validate( + layer, name, 'pre_hash_post_process', attribute_type=str, default=False + ) or error error = self.validate( layer, name, 'post_process', attribute_type=str, default=False ) or error diff --git a/tilecloud_chain/generate.py b/tilecloud_chain/generate.py index 31cf75dc..26d12111 100644 --- a/tilecloud_chain/generate.py +++ b/tilecloud_chain/generate.py @@ -187,7 +187,9 @@ def add_elapsed_togenerate(metatile): gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) self.count_tiles = gene.counter() - gene.process() + + if 'pre_hash_post_process' in gene.layer: + gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) @@ -205,6 +207,8 @@ def add_elapsed_togenerate(metatile): count=count_tiles_dropped, )) + gene.process() + if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) From 979a97ac70df595263ff1d4bc4d025cc845a2f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 10 Apr 2014 16:19:38 +0200 Subject: [PATCH 2/3] Close file descriptors instance of unlink file names --- tilecloud_chain/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index d3f204fc..467f4e2f 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -1456,7 +1456,7 @@ def __init__(self, config, options): def __call__(self, tile): if tile and tile.data: - name_in = tempfile.mkstemp()[1] + fd_in, name_in = tempfile.mkstemp() file_in = open(name_in, 'wb') file_in.write(tile.data) file_in.close() @@ -1476,7 +1476,7 @@ def __call__(self, tile): args.append(cmd['arg']['quiet']) if cmd['need_out']: - name_out = tempfile.mkstemp()[1] + fd_out, name_out = tempfile.mkstemp() os.unlink(name_out) else: # pragma: no cover name_out = name_in @@ -1499,13 +1499,14 @@ def __call__(self, tile): return tile if cmd['need_out']: - os.unlink(name_in) + os.close(fd_in) name_in = name_out + fd_in = fd_out - file_out = open(name_out, 'rb') + file_out = open(name_in, 'rb') tile.data = file_out.read() file_out.close() - os.unlink(name_out) + os.close(fd_in) return tile From b9563fa927b91b6e7fc274d9506b23bab92123a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 10 Apr 2014 17:52:12 +0200 Subject: [PATCH 3/3] Update the tests --- tilecloud_chain/tests/test_controller.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tilecloud_chain/tests/test_controller.py b/tilecloud_chain/tests/test_controller.py index f2f129d0..57ed39a3 100644 --- a/tilecloud_chain/tests/test_controller.py +++ b/tilecloud_chain/tests/test_controller.py @@ -1453,6 +1453,7 @@ def test_apache_s3(self): name: all params: {} post_process: false + pre_hash_post_process: false px_buffer: false type: wms url: http://localhost/mapserv @@ -1478,6 +1479,7 @@ def test_apache_s3(self): name: line params: {PARAM: value} post_process: false + pre_hash_post_process: false px_buffer: false type: wms url: http://localhost/mapserv @@ -1500,6 +1502,7 @@ def test_apache_s3(self): name: mapnik output_format: png post_process: false + pre_hash_post_process: false px_buffer: false type: mapnik wmts_style: default @@ -1526,6 +1529,7 @@ def test_apache_s3(self): name: mapnik_grid output_format: grid post_process: false + pre_hash_post_process: false px_buffer: false resolution: 16 type: mapnik @@ -1551,6 +1555,7 @@ def test_apache_s3(self): name: mapnik_grid_drop output_format: grid post_process: false + pre_hash_post_process: false px_buffer: false resolution: 16 type: mapnik @@ -1575,6 +1580,7 @@ def test_apache_s3(self): name: point params: {} post_process: false + pre_hash_post_process: false px_buffer: false sqs: {queue: sqs_point, region: eu-west-1} type: wms @@ -1599,6 +1605,7 @@ def test_apache_s3(self): name: point_hash_no_meta params: {} post_process: false + pre_hash_post_process: false px_buffer: false type: wms url: http://localhost/mapserv @@ -1625,6 +1632,7 @@ def test_apache_s3(self): name: point_hash params: {} post_process: false + pre_hash_post_process: false px_buffer: false type: wms url: http://localhost/mapserv @@ -1650,6 +1658,7 @@ def test_apache_s3(self): name: point_px_buffer params: {} post_process: false + pre_hash_post_process: false px_buffer: 100.0 type: wms url: http://localhost/mapserv @@ -1675,6 +1684,7 @@ def test_apache_s3(self): name: polygon params: {} post_process: false + pre_hash_post_process: false px_buffer: false type: wms url: http://localhost/mapserv @@ -1700,6 +1710,7 @@ def test_apache_s3(self): name: polygon2 params: {} post_process: false + pre_hash_post_process: false px_buffer: false type: wms url: http://localhost/mapserv