Print statistics

Add quiet option
Fix Mapnik Grid drop

CHANGES.rst

@@ -43,6 +43,8 @@ More informations in `Configure geom/sql <https://github.com/sbrunner/tilecloud-
 
 8. Now we can creates legends, see `Legends <https://github.com/sbrunner/tilecloud-chain/blob/master/README.rst#legends>`_
 
+9. Now the tiles generation display generation statistics at the ends.
+
 
 Release 0.6
 -----------

README.rst

@@ -697,7 +697,9 @@ The generation can be deported on an external host.
 Other useful options
 ---------------------
 
-``--verbose`` or ``-v``: used to display info message.
+``--quiet`` or ``-q``: used to display only errors.
+
+``--verbose`` or ``-v``: used to display info messages.
 
 ``--debug`` or ``-d``: used to display debug message, pleas use this option to report issue.
 With the debug mode we don't catch exceptions, and we don't log time messages.

tilecloud_chain/__init__.py

@@ -89,6 +89,10 @@ def add_comon_options(parser, tile_pyramid=True, no_geom=True):
         dest='cache', metavar="NAME",
         help='The cache name to use'
     )
+    parser.add_argument(
+        '-q', '--quiet', default=False, action="store_true",
+        help='Display only errors.'
+    )
     parser.add_argument(
         '-v', '--verbose', default=False, action="store_true",
         help='Display info message.'
@@ -115,8 +119,6 @@ class TileGeneration:
     geom = None
 
     def __init__(self, config_file, options=None, layer_name=None):
-        level = logging.WARNING
-
         if options is not None:
             if not hasattr(options, 'bbox'):
                 options.bbox = None
@@ -131,11 +133,12 @@ def __init__(self, config_file, options=None, layer_name=None):
             if not hasattr(options, 'geom'):
                 options.geom = True
 
-        if options and options.verbose and options.debug:  # pragma: no cover
-            exit("Debug and verbose options can't be used together")
-        if options and options.verbose:
+        level = logging.WARNING
+        if options and options.quiet:
+            level = logging.ERROR
+        elif options and options.verbose:
             level = logging.INFO
-        elif options and options.debug:  # pragma: no cover
+        elif options and options.debug:
             level = logging.DEBUG
         logging.basicConfig(
             format='%(levelname)s:%(name)s:%(funcName)s:%(message)s',
@@ -1087,11 +1090,12 @@ class HashDropper:
     The ``store`` is used to delete the empty tiles.
     """
 
-    def __init__(self, size, sha1code, store=None, queue_store=None):
+    def __init__(self, size, sha1code, store=None, queue_store=None, count=None):
         self.size = size
         self.sha1code = sha1code
         self.store = store
         self.queue_store = queue_store
+        self.count = count
 
     def __call__(self, tile):
         if len(tile.data) != self.size or \
@@ -1112,6 +1116,9 @@ def __call__(self, tile):
             elif self.queue_store is not None:  # pragma: no cover
                 self.queue_store.delete_one(tile)
 
+            if self.count:
+                self.count()
+
             return None
 
 
@@ -1177,8 +1184,11 @@ class DropEmpty:
     """
 
     def __call__(self, tile):
-        if not tile or not tile.data:
-            return None  # pragma: no cover
+        if not tile or not tile.data:  # pragma: no cover
+            logger.error("The tile: %(tilecoord)s is empty")
+            if 'error_file' in self.config['generation']:
+                self.log_tiles_error(tilecoord=tile.tilecoord, error='The tile is empty')
+            return None
         else:
             return tile
 

tilecloud_chain/amazon.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 
 import os
 import sys

tilecloud_chain/cost.py

@@ -8,6 +8,7 @@
 from tilecloud import Tile, TileStore, consume
 
 from tilecloud_chain import TileGeneration, add_comon_options
+from tilecloud_chain.format import duration_format
 
 
 logger = logging.getLogger(__name__)
@@ -52,8 +53,7 @@ def main():
         print
         print "===== GLOBAL ====="
         print "Total number of tiles: %i" % all_tiles
-        print 'Total generation time: %d %d:%02d:%02d [d h:mm:ss]' % \
-            (all_time.days, all_time.seconds / 3600, all_time.seconds % 3600 / 60, all_time.seconds % 60)
+        print 'Total generation time: %s [d h:mm:ss]' % (duration_format(all_time))
         print 'Total generation cost: %0.2f [$]' % all_price
     print
     print 'S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024))
@@ -248,8 +248,7 @@ def count_tile(tile):
 
         all_time += time
         td = timedelta(milliseconds=time)
-        print "Time to generate: %d %d:%02d:%02d [d h:mm:ss]" % \
-            (td.days, td.seconds / 3600, td.seconds % 3600 / 60, td.seconds % 60)
+        print "Time to generate: %s [d h:mm:ss]" % (duration_format(td))
         c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0
         price += c
         print 'S3 PUT: %0.2f [$]' % c
@@ -270,8 +269,7 @@ def count_tile(tile):
     print
     td = timedelta(milliseconds=all_time)
     print "Number of tiles: %i" % all_tiles
-    print 'Generation time: %d %d:%02d:%02d [d h:mm:ss]' % \
-        (td.days, td.seconds / 3600, td.seconds % 3600 / 60, td.seconds % 60)
+    print 'Generation time: %s [d h:mm:ss]' % (duration_format(td))
     print 'Generation cost: %0.2f [$]' % price
 
     return (all_size, td, price, all_tiles)

tilecloud_chain/format.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+
+def size_format(number):
+    for unit in ['o', 'Kio', 'Mio', 'Gio', 'Tio']:
+        if number < 1024.0:
+            if number < 10:
+                return "%.1f %s" % (number, unit)
+            else:
+                return "%.0f %s" % (number, unit)
+        number /= 1024.0
+
+
+def duration_format(duration):
+    hours, remainder = divmod(duration.seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    if duration.days > 0:
+        return '%i %i:%02i:%02i' % (duration.days, hours, minutes, seconds)
+    else:
+        return '%i:%02i:%02i' % (hours, minutes, seconds)