fixed loading of timestamp from missing file

mapproxy/seed/config.py

@@ -15,6 +15,7 @@
 
 from __future__ import with_statement
 
+import os
 import sys
 import time
 import operator
@@ -24,10 +25,9 @@
 from mapproxy.config.coverage import load_coverage
 from mapproxy.srs import SRS
 from mapproxy.util import (
-    memoize, 
-    timestamp_from_isodate, 
-    timestamp_before, 
-    timestamp_from_file
+    memoize,
+    timestamp_from_isodate,
+    timestamp_before,
 )
 from mapproxy.util.coverage import MultiCoverage, BBOXCoverage, GeomCoverage
 from mapproxy.util.yaml import load_yaml_file, YAMLError
@@ -48,7 +48,7 @@ def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
         conf = load_yaml_file(seed_conf_filename)
     except YAMLError, ex:
         raise SeedConfigurationError(ex)
-    
+
     if 'views' in conf:
         # TODO: deprecate old config
         seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
@@ -72,7 +72,7 @@ def __init__(self, seed_conf, mapproxy_conf):
         self.seed_tasks = []
         self.cleanup_tasks = []
         self._init_tasks()
-        
+
     def _init_tasks(self):
         for cache_name, options in self.conf['seeds'].iteritems():
             remove_before = None
@@ -92,10 +92,10 @@ def _init_tasks(self):
                 cache_srs = view_conf.get('srs', None)
                 if cache_srs is not None:
                     cache_srs = [SRS(s) for s in cache_srs]
-            
+
                 level = view_conf.get('level', None)
                 assert len(level) == 2
-                
+
                 for grid, tile_mgr in caches.iteritems():
                     if cache_srs and grid.srs not in cache_srs: continue
                     md = dict(name=view, cache_name=cache_name, grid_name=self.grids[grid])
@@ -106,18 +106,18 @@ def _init_tasks(self):
                         seed_coverage = coverage.transform_to(grid.srs)
                     else:
                         seed_coverage = BBOXCoverage(grid.bbox, grid.srs)
-                    
+
                     self.seed_tasks.append(SeedTask(md, tile_mgr, levels, remove_before, seed_coverage))
-            
+
                     if remove_before:
                         levels = range(grid.levels)
                         complete_extent = bool(coverage)
                         self.cleanup_tasks.append(CleanupTask(md, tile_mgr, levels, remove_before,
                             seed_coverage, complete_extent=complete_extent))
-                        
+
     def seed_tasks_names(self):
         return self.conf['seeds'].keys()
-    
+
     def cleanup_tasks_names(self):
         return self.conf['seeds'].keys()
 
@@ -126,7 +126,7 @@ def seeds(self, names=None):
             return self.seed_tasks
         else:
             return [t for t in self.seed_tasks if t.md['name'] in names]
-    
+
     def cleanups(self, names=None):
         if names is None:
             return self.cleanup_tasks
@@ -144,9 +144,9 @@ def coverage(self, name):
         coverage_conf = self.conf['coverages'].get(name)
         if coverage_conf is None:
             raise ValueError('no coverage %s configured' % name)
-        
+
         return load_coverage(coverage_conf)
-    
+
     def cache(self, cache_name):
         cache = {}
         if cache_name not in self.mapproxy_conf.caches:
@@ -156,13 +156,13 @@ def cache(self, cache_name):
             grid_name = self.grids[tile_grid]
             cache[grid_name] = tile_mgr
         return cache
-    
+
     def seed_tasks_names(self):
         return self.conf.get('seeds', {}).keys()
-    
+
     def cleanup_tasks_names(self):
         return self.conf.get('cleanups', {}).keys()
-    
+
     def seeds(self, names=None):
         """
         Return seed tasks.
@@ -174,7 +174,7 @@ def seeds(self, names=None):
             for task in seed_conf.seed_tasks():
                 tasks.append(task)
         return tasks
-        
+
     def cleanups(self, names=None):
         """
         Return cleanup tasks.
@@ -208,10 +208,10 @@ def _coverages(self):
             else:
                 coverage = MultiCoverage(coverages)
         return coverage
-    
+
     def _grids(self, caches):
         grids = []
-        
+
         if 'grids' in self.conf:
             # grids available for all caches
             available_grids = reduce(operator.and_, (set(cache) for cache in caches.values()))
@@ -230,11 +230,11 @@ def _grids(self, caches):
                         raise SeedConfigurationError('caches in same seed task require identical grids')
             grids = list(last or [])
         return grids
-    
+
     def _caches(self):
         """
         Returns a dictionary with all caches for this seed.
-        
+
         e.g.: {'seed1': {'grid1': tilemanager1, 'grid2': tilemanager2}}
         """
         caches = {}
@@ -246,7 +246,7 @@ def _caches(self):
 class SeedConfiguration(ConfigurationBase):
     def __init__(self, name, conf, seeding_conf):
         ConfigurationBase.__init__(self, name, conf, seeding_conf)
-        
+
         self.refresh_timestamp = None
         if 'refresh_before' in self.conf:
             self.refresh_timestamp = before_timestamp_from_options(self.conf['refresh_before'])
@@ -266,27 +266,27 @@ def seed_tasks(self):
                     levels = self.levels.for_grid(grid)
                 else:
                     levels = list(xrange(0, grid.levels))
-                
+
                 if not tile_manager.cache.supports_timestamp:
                     if self.refresh_timestamp:
                         # remove everything
                         self.refresh_timestamp = 0
-                
+
                 md = dict(name=self.name, cache_name=cache_name, grid_name=grid_name)
                 yield SeedTask(md, tile_manager, levels, self.refresh_timestamp, coverage)
 
 class CleanupConfiguration(ConfigurationBase):
     def __init__(self, name, conf, seeding_conf):
         ConfigurationBase.__init__(self, name, conf, seeding_conf)
         self.init_time = time.time()
-        
+
         if 'remove_before' in self.conf:
             self.remove_timestamp = before_timestamp_from_options(self.conf['remove_before'])
         else:
             # use now as remove_before date. this should not remove
             # fresh seeded tiles, since this should be configured before seeding
             self.remove_timestamp = self.init_time
-    
+
     def cleanup_tasks(self):
         for grid_name in self.grids:
             for cache_name, cache in self.caches.iteritems():
@@ -302,7 +302,7 @@ def cleanup_tasks(self):
                     levels = self.levels.for_grid(grid)
                 else:
                     levels = list(xrange(0, grid.levels))
-                
+
                 if not tile_manager.cache.supports_timestamp:
                     # for caches without timestamp support (like MBTiles)
                     if self.remove_timestamp is self.init_time:
@@ -312,9 +312,9 @@ def cleanup_tasks(self):
                         raise SeedConfigurationError("cleanup does not support remove_before for '%s'"
                             " because cache '%s' does not support timestamps" % (self.name, cache_name))
                 md = dict(name=self.name, cache_name=cache_name, grid_name=grid_name)
-                yield CleanupTask(md, tile_manager, levels, self.remove_timestamp, 
+                yield CleanupTask(md, tile_manager, levels, self.remove_timestamp,
                     coverage=coverage, complete_extent=complete_extent)
-    
+
 
 def levels_from_options(conf):
     levels = conf.get('levels')
@@ -349,10 +349,10 @@ def before_timestamp_from_options(conf):
     if 'mtime' in conf:
         datasource = abspath(conf['mtime'])
         try:
-            return timestamp_from_file(datasource)
-        except OSError:
-            raise SeedConfiguration(
-                "can't parse last modified time from file '%s'." % (datasource, ))
+            return os.path.getmtime(datasource)
+        except OSError, ex:
+            raise SeedConfigurationError(
+                "can't parse last modified time from file '%s'." % (datasource, ), ex)
     deltas = {}
     for delta_type in ('weeks', 'days', 'hours', 'minutes'):
         deltas[delta_type] = conf.get(delta_type, 0)
@@ -362,7 +362,7 @@ def before_timestamp_from_options(conf):
 class LevelsList(object):
     def __init__(self, levels=None):
         self.levels = levels
-    
+
     def for_grid(self, grid):
         uniqe_valid_levels = set(l for l in self.levels if 0 <= l <= (grid.levels-1))
         return sorted(uniqe_valid_levels)
@@ -377,32 +377,32 @@ def for_grid(self, grid):
             start = 0
         if stop is None:
             stop = 999
-        
+
         stop = min(stop, grid.levels-1)
-        
+
         return list(xrange(start, stop+1))
 
 class LevelsResolutionRange(object):
     def __init__(self, res_range=None):
-        self.res_range = res_range  
+        self.res_range = res_range
     def for_grid(self, grid):
         start, stop = self.res_range
         if start is None:
             start = 0
         else:
             start = grid.closest_level(start)
-        
+
         if stop is None:
             stop = grid.levels-1
         else:
             stop = grid.closest_level(stop)
-        
+
         return list(xrange(start, stop+1))
-        
+
 class LevelsResolutionList(object):
     def __init__(self, resolutions=None):
         self.resolutions = resolutions
-    
+
     def for_grid(self, grid):
         levels = set(grid.closest_level(res) for res in self.resolutions)
         return sorted(levels)

mapproxy/test/unit/test_seed.py

@@ -14,20 +14,25 @@
 # limitations under the License.
 
 from __future__ import with_statement, division
+
+import os
+import time
 import cPickle as pickle
+
 from mapproxy.seed.seeder import TileWalker, SeedTask, SeedProgress
 from mapproxy.cache.tile import TileManager
 from mapproxy.source.tile import TiledSource
 from mapproxy.grid import tile_grid_for_epsg
 from mapproxy.grid import TileGrid
 from mapproxy.srs import SRS
 from mapproxy.util.coverage import BBOXCoverage, GeomCoverage
+from mapproxy.seed.config import before_timestamp_from_options, SeedConfigurationError
 from mapproxy.seed.config import LevelsList, LevelsRange, LevelsResolutionList, LevelsResolutionRange
 from mapproxy.seed.util import ProgressStore
 from mapproxy.test.helper import TempFile
 
 from collections import defaultdict
-from nose.tools import eq_
+from nose.tools import eq_, assert_almost_equal, raises
 from nose.plugins.skip import SkipTest
 
 try:
@@ -224,3 +229,29 @@ def test_load_broken(self):
 
             store = ProgressStore(tmp)
             assert store.status == {}
+
+
+class TestRemovebreforeTimetamp(object):
+    def test_from_time(self):
+        assert_almost_equal(before_timestamp_from_options({'time': '2012-12-01T20:12:00'}), 1354389120.0, -2)
+
+    def test_from_mtime(self):
+        with TempFile() as tmp:
+            os.utime(tmp, (12376512, 12376512))
+            eq_(before_timestamp_from_options({'mtime': tmp}), 12376512)
+
+    @raises(SeedConfigurationError)
+    def test_from_mtime_missing_file(self):
+        before_timestamp_from_options({'mtime': '/tmp/does-not-exist-at-all,really'})
+
+    def test_from_empty(self):
+        assert_almost_equal(
+            before_timestamp_from_options({}),
+            time.time(), -1
+        )
+
+    def test_from_delta(self):
+        assert_almost_equal(
+            before_timestamp_from_options({'minutes': 15}) + 60 * 15,
+            time.time(), -1
+        )

mapproxy/util/__init__.py

@@ -154,12 +154,6 @@ def timestamp_from_isodate(isodate):
         date = datetime.datetime.strptime(isodate, "%Y-%m-%dT%H:%M:%S")
     return time.mktime(date.timetuple())
 
-def timestamp_from_file(path):
-    if not os.path.exists(path):
-        raise OSError
-    # returns time since epoch 1970
-    return os.path.getmtime(path)
-
 def cleanup_directory(directory, before_timestamp, remove_empty_dirs=True,
                       file_handler=None):
     if file_handler is None: