Add nb error

camptocamp · Nov 3, 2013 · 2c36d99 · 2c36d99
1 parent db6467a
commit 2c36d99
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 4 deletions.
diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py
@@ -41,7 +41,7 @@
 from tilecloud.store.filesystem import FilesystemTileStore
 from tilecloud.layout.wmts import WMTSTileLayout
 from tilecloud.filter.logger import Logger
-from tilecloud.filter.error import LogErrors, MaximumConsecutiveErrors, DropErrors
+from tilecloud.filter.error import LogErrors, MaximumConsecutiveErrors
 
 
 logger = logging.getLogger('tilecloud_chain')
@@ -128,10 +128,11 @@ def get_tile_matrix_identifier(grid, resolution=None, zoom=None):
 
 
 class TileGeneration:
-    geom = None
 
     def __init__(self, config_file, options=None, layer_name=None):
         self.close_actions = []
+        self.geom = None
+        self.error = 0
 
         if options is not None:
             if not hasattr(options, 'bbox'):
@@ -971,6 +972,7 @@ def add_error_filters(self):
             logger, logging.ERROR,
             "Error in tile: %(tilecoord)s, %(error)r"
         ))
+
         if 'error_file' in self.config['generation']:
 
             def do(tile):
@@ -981,7 +983,13 @@ def do(tile):
         if 'maxconsecutive_errors' in self.config['generation']:
             self.tilestream = imap(MaximumConsecutiveErrors(
                 self.config['generation']['maxconsecutive_errors']), self.tilestream)
-        self.ifilter(DropErrors())
+
+        def drop_count(tile):
+            if tile and tile.error:
+                self.error += 1
+                return None
+            return tile
+        self.ifilter(drop_count)
 
     def init_tilecoords(self):
         resolutions = self.layer['grid_ref']['resolutions']

diff --git a/tilecloud_chain/copy_.py b/tilecloud_chain/copy_.py
@@ -66,7 +66,7 @@ def main():
 
     gene = TileGeneration(options.config, options)
 
-    if (options.layer):
+    if (options.layer):  # pragma: no cover
         copy = Copy()
         copy.copy(options, gene, options.layer)
     else:

diff --git a/tilecloud_chain/generate.py b/tilecloud_chain/generate.py
@@ -263,6 +263,7 @@ def __call__(self, tile):
 %sNb generated tiles: %i
 Nb tiles dropped: %i
 Nb tiles stored: %i
+Nb error: %i
 Total time: %s
 Total size: %s
 Time per tiles: %i ms
@@ -279,6 +280,7 @@ def __call__(self, tile):
                         nb_tiles,
                         count_tiles_dropped.nb,
                         count_tiles.nb,
+                        gene.error,
                         duration_format(gene.duration),
                         size_format(count_tiles.size),
                         (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,

diff --git a/tilecloud_chain/tests/test_generate.py b/tilecloud_chain/tests/test_generate.py
@@ -134,6 +134,7 @@ def test_test_all(self, l):
 Nb generated tiles: 40
 Nb tiles dropped: 39
 Nb tiles stored: 1
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 733 o
 Time per tiles: [0-9]+ ms
@@ -143,6 +144,7 @@ def test_test_all(self, l):
 Nb generated tiles: 1
 Nb tiles dropped: 0
 Nb tiles stored: 1
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [45][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -174,6 +176,7 @@ def test_test_dimensions(self, l):
 Nb generated tiles: 40
 Nb tiles dropped: 39
 Nb tiles stored: 1
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 733 o
 Time per tiles: [0-9]+ ms
@@ -183,6 +186,7 @@ def test_test_dimensions(self, l):
 Nb generated tiles: 1
 Nb tiles dropped: 0
 Nb tiles stored: 1
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [45][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -260,6 +264,7 @@ def test_multigeom(self, l):
 Nb generated tiles: 51
 Nb tiles dropped: 0
 Nb tiles stored: 51
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [34][0-9] Kio
 Time per tiles: [0-9]+ ms
@@ -290,6 +295,7 @@ def test_zoom_identifier(self, l):
 Nb generated tiles: 42
 Nb tiles dropped: 41
 Nb tiles stored: 1
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 389 o
 Time per tiles: [0-9]+ ms
@@ -312,6 +318,7 @@ def test_zoom_identifier(self, l):
 Nb generated tiles: 34
 Nb tiles dropped: 33
 Nb tiles stored: 1
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 517 o
 Time per tiles: [0-9]+ ms
@@ -334,6 +341,7 @@ def test_zoom_identifier(self, l):
 Nb generated tiles: 4
 Nb tiles dropped: 3
 Nb tiles stored: 1
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 676 o
 Time per tiles: [0-9]+ ms
@@ -364,6 +372,7 @@ def test_empty_bbox(self, l):
 Nb generated tiles: 0
 Nb tiles dropped: 0
 Nb tiles stored: 0
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 0.0 o
 Time per tiles: [0-9]+ ms
@@ -404,6 +413,7 @@ def test_zoom(self, l):
 Nb generated tiles: 64
 Nb tiles dropped: 62
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -436,6 +446,7 @@ def test_zoom_range(self, l):
 Nb generated tiles: 320
 Nb tiles dropped: 314
 Nb tiles stored: 6
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 2.7 Kio
 Time per tiles: [0-9]+ ms
@@ -469,6 +480,7 @@ def test_no_zoom(self, l):
 Nb generated tiles: 384
 Nb tiles dropped: 376
 Nb tiles stored: 8
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 3.6 Kio
 Time per tiles: [0-9]+ ms
@@ -502,6 +514,7 @@ def test_py_buffer(self, l):
 Nb generated tiles: 384
 Nb tiles dropped: 378
 Nb tiles stored: 6
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 2.7 Kio
 Time per tiles: [0-9]+ ms
@@ -534,6 +547,7 @@ def test_zoom_list(self, l):
 Nb generated tiles: 320
 Nb tiles dropped: 314
 Nb tiles stored: 6
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 2.7 Kio
 Time per tiles: [0-9]+ ms
@@ -560,6 +574,7 @@ def test_layer_bbox(self, l):
 Nb generated tiles: 12
 Nb tiles dropped: 0
 Nb tiles stored: 12
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [0-9.]+ Kio
 Time per tiles: [0-9.]+ ms
@@ -582,6 +597,7 @@ def test_layer_bbox(self, l):
 Nb generated tiles: 2
 Nb tiles dropped: 0
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 1.[6-9] Kio
 Time per tiles: [0-9]+ ms
@@ -604,6 +620,7 @@ def test_layer_bbox(self, l):
 Nb generated tiles: 2
 Nb tiles dropped: 0
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 1.[6-9] Kio
 Time per tiles: [0-9]+ ms
@@ -625,6 +642,7 @@ def test_layer_bbox(self, l):
 Nb generated tiles: 2
 Nb tiles dropped: 0
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 1.[6-9] Kio
 Time per tiles: [0-9]+ ms
@@ -655,6 +673,7 @@ def test_hash_generation(self, l):
 Nb generated tiles: 64
 Nb tiles dropped: 62
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 910 o
 Time per tiles: [0-9]+ ms
@@ -681,6 +700,7 @@ def test_mapnik(self, l):
 Nb generated tiles: 12
 Nb tiles dropped: 0
 Nb tiles stored: 12
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 9.7 Kio
 Time per tiles: [0-9]+ ms
@@ -707,6 +727,7 @@ def test_mapnik_grid(self, l):
 Nb generated tiles: 12
 Nb tiles dropped: 0
 Nb tiles stored: 12
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 4.5 Kio
 Time per tiles: [0-9]+ ms
@@ -747,6 +768,7 @@ def test_mapnik_grid_drop(self, l):
 Nb generated tiles: 12
 Nb tiles dropped: 10
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: 768 o
 Time per tiles: [0-9]+ ms
@@ -863,6 +885,7 @@ def test_delete_meta(self):
 Nb generated tiles: 247
 Nb tiles dropped: 245
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -895,6 +918,7 @@ def test_delete_no_meta(self):
 Nb generated tiles: 247
 Nb tiles dropped: 245
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -939,6 +963,7 @@ def test_error_file(self):
 Nb generated tiles: 128
 Nb tiles dropped: 126
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms

diff --git a/tilecloud_chain/tests/test_serve.py b/tilecloud_chain/tests/test_serve.py
@@ -179,6 +179,7 @@ def test_serve_kvp(self, l):
 Nb generated tiles: 64
 Nb tiles dropped: 62
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -722,6 +723,7 @@ def test_mbtiles_rest(self, l):
 Nb generated tiles: 64
 Nb tiles dropped: 62
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -820,6 +822,7 @@ def test_bsddb_rest(self, l):
 Nb generated tiles: 64
 Nb tiles dropped: 62
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms
@@ -992,6 +995,7 @@ def test_wsgi(self, l):
 Nb generated tiles: 64
 Nb tiles dropped: 62
 Nb tiles stored: 2
+Nb error: 0
 Total time: [0-9]+:[0-9][0-9]:[0-9][0-9]
 Total size: [89][0-9][0-9] o
 Time per tiles: [0-9]+ ms