Merge pull request #175 from sbrunner/fix-drop-empty

Fix Drop Empty

tilecloud_chain/__init__.py

@@ -119,6 +119,8 @@ class TileGeneration:
     geom = None
 
     def __init__(self, config_file, options=None, layer_name=None):
+        self.close_actions = []
+
         if options is not None:
             if not hasattr(options, 'bbox'):
                 options.bbox = None
@@ -596,13 +598,22 @@ def get_store(self, cache, layer, dimensions=None, read_only=False):
             ) + '.bsddb'
             if not os.path.exists(os.path.dirname(filename)):
                 os.makedirs(os.path.dirname(filename))
+            db = bsddb.hashopen(
+                filename,
+                # and os.path.exists(filename) to avoid error on non existing file
+                'r' if read_only and os.path.exists(filename) else 'c'
+            )
+
+            class Close:
+                def __call__(self):
+                    self.db.close()
+
+            ca = Close()
+            ca.db = db
+            self.close_actions.append(ca)
+
             cache_tilestore = BSDDBTileStore(
-                bsddb.hashopen(
-                    filename,
-                    # and os.path.exists(filename) to avoid error on non existing file
-                    'r' if read_only and os.path.exists(filename) else 'c'
-                ),
-                content_type=layer['mime_type'],
+                db, content_type=layer['mime_type'],
             )
         elif cache['type'] == 'filesystem':
             # on filesystem
@@ -1227,11 +1238,16 @@ class DropEmpty:
     Create a filter for dropping all tiles with errors.
     """
 
+    def __init__(self, gene):
+        self.gene = gene
+
     def __call__(self, tile):
         if not tile or not tile.data:  # pragma: no cover
-            logger.error("The tile: %(tilecoord)s is empty")
-            if 'error_file' in self.config['generation']:
-                self.log_tiles_error(tilecoord=tile.tilecoord, error='The tile is empty')
+            logger.error("The tile: %(tilecoord)s is empty" % {
+                'tilecoord': tile.tilecoord if tile else 'not defined'
+            })
+            if 'error_file' in self.gene.config['generation'] and tile:
+                self.gene.log_tiles_error(tilecoord=tile.tilecoord, error='The tile is empty')
             return None
         else:
             return tile

tilecloud_chain/generate.py

@@ -255,7 +255,7 @@ def add_elapsed_togenerate(metatile):
 
         if options.role in ('local', 'slave'):
             gene.add_error_filters(logger)
-            gene.ifilter(DropEmpty())
+            gene.ifilter(DropEmpty(gene))
             gene.imap(self.inc_tiles_size)
 
             if options.time:
@@ -335,6 +335,9 @@ def __call__(self, tile):
                         self.tiles_size / self.nb_tiles_stored if self.nb_tiles_stored != 0 else -1
                     )
 
+        for ca in gene.close_actions:
+            ca()
+
         if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'):
             cache_tilestore.connection.close()
 

tilecloud_chain/tests/test_serve.py

@@ -731,7 +731,7 @@ def test_mbtiles_rest(self, l):
         )
         # use delete to don't delete the repository
         self.assert_tiles_generated_deleted(
-            cmd='./buildout/bin/generate_controller --capabilities -c tilegeneration/test-serve.yaml',
+            cmd='./buildout/bin/generate_controller -d --capabilities -c tilegeneration/test-serve.yaml',
             main_func=controller.main,
             directory="/tmp/tiles/mbtiles/",
             tiles_pattern='1.0.0/%s',