Permalink
Browse files

Merge branch 'master' into single-thread-mapnik-provider

Conflicts:
	scripts/tilestache-compose.py
  • Loading branch information...
2 parents 4f4863e + 991e0ea commit fa0184af46b6444cbabf8ce144f79576cb8999af Michal Migurski committed Feb 13, 2012
View
@@ -692,6 +692,7 @@
"write cache": …,
"bounds": { … },
"allowed origin": …,
+ "maximum cache age": …,
"jpeg options": …,
"png options": …
}
@@ -781,6 +782,16 @@
security headache, use a value of <samp>"*"</samp> for this.
</dd>
+ <dt>maximum cache age</dt>
+ <dd>
+ An optional number of seconds used to control behavior of downstream caches.
+ Causes TileStache responses to include
+ <a href="http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9">Cache-Control</a>
+ and <a href="http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21">Expires</a>
+ HTTP response headers. Useful when TileStache is itself hosted behind an HTTP
+ cache such as Squid, Cloudfront, or Akamai.
+ </dd>
+
<dt>jpeg options</dt>
<dd>
An optional dictionary of JPEG creation options, passed through
@@ -1636,8 +1647,8 @@
</p>
<p>
-A cache must provide all of these four methods: <code>lock</code>,
-<code>unlock</code>, <code>read</code>, and <code>save</code>.
+A cache must provide all of these five methods: <code>lock</code>,
+<code>unlock</code>, <code>remove</code>, <code>read</code>, and <code>save</code>.
</p>
<p>
@@ -1688,6 +1699,10 @@
# unlock a tile
raise NotImplementedError
+ def remove(self, layer, coord, format):
+ # remove a tile
+ raise NotImplementedError
+
def read(self, layer, coord, format):
# return raw tile content from cache
raise NotImplementedError
View
@@ -1,3 +1,38 @@
+2012-02-06: 1.25.1
+- Fixed tilestache-clean.py script to not complain on unsupported -e flag.
+
+2012-02-01: 1.25.0
+- Added "ALL" ("ALL LAYERS") option to tilestache-clean.py script.
+
+2012-01-30: 1.24.1
+- Fixed an error in WSGITileServer._response().
+
+2012-01-25: 1.24.0
+- Added support for HTTP Expires and Cache-Control headers with per-layer maximum cache age.
+- Merged Per Liedman's fox for disk cache lock directory removal bug.
+
+2012-01-23: 1.23.4
+- Fixed a bug in Multi Cache provider remove().
+
+2012-01-12: 1.23.3
+- Changed import delay from 1.23.2 to a specific conditional around sqlite3 for Heroku.
+
+2012-01-11: 1.23.2
+- Delayed import of MBTiles and Vector providers until last minute.
+
+2012-01-10: 1.23.1
+- Added Nikolai's httplib patch in Proxy provider.
+
+2012-01-10: 1.23.0
+- Added new tilestache-clean.py script and required remove() method for caches.
+
+2012-01-01: 1.22.0
+- Fixed inconsistencies with cache-ignore setting when using tilestache-compose.py.
+- Fixed use of write-cache setting in TileStache.Layer.
+- Adding configuration file's directory to sys.path in Config.buildConfiguration().
+- Stopped adding current working directory to sys.path for new classes.
+- Added new TileStache.Goodies.Providers.MapnikGrid:Provider by github user danzel.
+
2011-12-25: 1.21.3
- Fixed a bug where I'm an idiot.
View
@@ -76,6 +76,7 @@ doc:
pydoc -w TileStache.Goodies.Providers.Composite
pydoc -w TileStache.Goodies.Providers.PostGeoJSON
pydoc -w TileStache.Goodies.Providers.SolrGeoJSON
+ pydoc -w TileStache.Goodies.Providers.MapnikGrid
pydoc -w TileStache.Goodies.Providers.MirrorOSM
pydoc -w TileStache.Goodies.Providers.Grid
pydoc -w TileStache.Goodies.Providers.GDAL
View
@@ -124,6 +124,14 @@ def unlock(self, layer, coord, format):
if self.logfunc:
self.logfunc('Test cache unlock: ' + name)
+ def remove(self, layer, coord, format):
+ """ Pretend to remove a cached tile.
+ """
+ name = self._description(layer, coord, format)
+
+ if self.logfunc:
+ self.logfunc('Test cache remove: ' + name)
+
def read(self, layer, coord, format):
""" Pretend to read a cached tile.
"""
@@ -253,8 +261,25 @@ def unlock(self, layer, coord, format):
Lock is implemented as an empty directory next to the tile file.
"""
lockpath = self._lockpath(layer, coord, format)
- os.rmdir(lockpath)
-
+
+ try:
+ os.rmdir(lockpath)
+ except OSError:
+ # Ok, someone else deleted it already
+ pass
+
+ def remove(self, layer, coord, format):
+ """ Remove a cached tile.
+ """
+ fullpath = self._fullpath(layer, coord, format)
+
+ try:
+ os.remove(fullpath)
+ except OSError, e:
+ # errno=2 means that the file does not exist, which is fine
+ if e.errno != 2:
+ raise
+
def read(self, layer, coord, format):
""" Read a cached tile.
"""
@@ -359,7 +384,13 @@ def unlock(self, layer, coord, format):
""" Release a cache lock for this tile in the first tier.
"""
return self.tiers[0].unlock(layer, coord, format)
-
+
+ def remove(self, layer, coord, format):
+ """ Remove a cached tile from every tier.
+ """
+ for (index, cache) in enumerate(self.tiers):
+ cache.remove(layer, coord, format)
+
def read(self, layer, coord, format):
""" Read a cached tile.
View
@@ -302,6 +302,9 @@ def _parseConfigfileLayer(layer_dict, config, dirpath):
if 'allowed origin' in layer_dict:
layer_kwargs['allowed_origin'] = str(layer_dict['allowed origin'])
+ if 'maximum cache age' in layer_dict:
+ layer_kwargs['max_cache_age'] = int(layer_dict['maximum cache age'])
+
if 'preview' in layer_dict:
preview_dict = layer_dict['preview']
View
@@ -22,6 +22,7 @@
"write cache": ...,
"bounds": { ... },
"allowed origin": ...,
+ "maximum cache age": ...,
"jpeg options": ...,
"png options": ...
}
@@ -53,6 +54,10 @@
header Access-Control-Allow-Origin, useful for when you need to provide
javascript direct access to response data such as GeoJSON or pixel values.
The header is part of a W3C working draft (http://www.w3.org/TR/cors/).
+- "maximum cache age" is an optional number of seconds used to control behavior
+ of downstream caches. Causes TileStache responses to include Cache-Control
+ and Expires HTTP response headers. Useful when TileStache is itself hosted
+ behind an HTTP cache such as Squid, Cloudfront, or Akamai.
- "jpeg options" is an optional dictionary of JPEG creation options, passed
through to PIL: http://www.pythonware.com/library/pil/handbook/format-jpeg.htm.
- "png options" is an optional dictionary of PNG creation options, passed
@@ -214,6 +219,9 @@ class Layer:
allowed_origin:
Value for the Access-Control-Allow-Origin HTTP response header.
+ max_cache_age:
+ Number of seconds that tiles from this layer may be cached by downstream clients.
+
preview_lat:
Starting latitude for slippy map layer preview, default 37.80.
@@ -226,7 +234,7 @@ class Layer:
preview_ext:
Tile name extension for slippy map layer preview, default "png".
"""
- def __init__(self, config, projection, metatile, stale_lock_timeout=15, cache_lifespan=None, write_cache=True, allowed_origin=None, preview_lat=37.80, preview_lon=-122.26, preview_zoom=10, preview_ext='png', bounds=None):
+ def __init__(self, config, projection, metatile, stale_lock_timeout=15, cache_lifespan=None, write_cache=True, allowed_origin=None, max_cache_age=None, preview_lat=37.80, preview_lon=-122.26, preview_zoom=10, preview_ext='png', bounds=None):
self.provider = None
self.config = config
self.projection = projection
@@ -236,6 +244,7 @@ def __init__(self, config, projection, metatile, stale_lock_timeout=15, cache_li
self.cache_lifespan = cache_lifespan
self.write_cache = write_cache
self.allowed_origin = allowed_origin
+ self.max_cache_age = max_cache_age
self.preview_lat = preview_lat
self.preview_lon = preview_lon
@@ -262,10 +271,8 @@ def name(self):
def doMetatile(self):
""" Return True if we have a real metatile and the provider is OK with it.
-
- self.write_cache == False will cause this to return False.
"""
- return self.metatile.isForReal() and hasattr(self.provider, 'renderArea') and self.write_cache
+ return self.metatile.isForReal() and hasattr(self.provider, 'renderArea')
def render(self, coord, format):
""" Render a tile for a coordinate, return PIL Image-like object.
@@ -326,7 +333,8 @@ def render(self, coord, format):
subtile.save(buff, format)
body = buff.getvalue()
- self.config.cache.save(body, self, other, format)
+ if self.write_cache:
+ self.config.cache.save(body, self, other, format)
if other == coord:
# the one that actually gets returned
@@ -122,7 +122,13 @@ def unlock(self, layer, coord, format):
(coord.row, coord.column, coord.zoom, format))
db.connection.commit()
db.connection.close()
-
+
+ def remove(self, layer, coord, format):
+ """ Remove a cached tile.
+ """
+ # TODO: write me
+ raise NotImplementedError('LimitedDisk Cache does not yet implement the .remove() method.')
+
def read(self, layer, coord, format):
""" Read a cached tile.
@@ -0,0 +1,103 @@
+""" Mapnik UTFGrid Provider.
+
+Takes the first layer from the given mapnik xml file and renders it as UTFGrid
+https://github.com/mapbox/mbtiles-spec/blob/master/1.1/utfgrid.md
+It can then be used for this:
+http://mapbox.github.com/wax/interaction-leaf.html
+Only works with mapnik2 (Where the Grid functionality was introduced)
+
+Use Sperical Mercator projection and the extension "json"
+
+Sample configuration:
+
+ "provider":
+ {
+ "class": "TileStache.Goodies.Providers.MapnikGrid:Provider",
+ "kwargs":
+ {
+ "mapfile": "mymap.xml",
+ "fields":["name", "address"],
+ "layer_index": 0,
+ "wrapper": "grid",
+ "scale": 4
+ }
+ }
+
+mapfile: the mapnik xml file to load the map from
+fields: The fields that should be added to the resulting grid json.
+layer_index: The index of the layer you want from your map xml to be rendered
+wrapper: If not included the json will be output raw, if included the json will be wrapped in "wrapper(JSON)" (for use with wax)
+scale: What to divide the tile pixel size by to get the resulting grid size. Usually this is 4.
+"""
+import json
+import mapnik2 as mapnik
+from TileStache.Geography import getProjectionByName
+
+class Provider:
+
+ def __init__(self, layer, mapfile, fields, layer_index=0, wrapper=None, scale=4):
+ """
+ """
+ self.mapnik = None
+ self.layer = layer
+ self.mapfile = mapfile
+ self.layer_index = layer_index
+ self.wrapper = wrapper
+ self.scale = scale
+ #De-Unicode the strings or mapnik gets upset
+ self.fields = list(str(x) for x in fields)
+
+ self.mercator = getProjectionByName('spherical mercator')
+
+ def renderTile(self, width, height, srs, coord):
+ """
+ """
+ if self.mapnik is None:
+ self.mapnik = mapnik.Map(0, 0)
+ mapnik.load_map(self.mapnik, str(self.mapfile))
+
+ nw = self.layer.projection.coordinateLocation(coord)
+ se = self.layer.projection.coordinateLocation(coord.right().down())
+ ul = self.mercator.locationProj(nw)
+ lr = self.mercator.locationProj(se)
+
+
+ self.mapnik.width = width
+ self.mapnik.height = height
+ self.mapnik.zoom_to_box(mapnik.Box2d(ul.x, ul.y, lr.x, lr.y))
+
+ # create grid as same size as map/image
+ grid = mapnik.Grid(width, height)
+ # render a layer to that grid array
+ mapnik.render_layer(self.mapnik, grid, layer=self.layer_index, fields=self.fields)
+ # then encode the grid array as utf, resample to 1/scale the size, and dump features
+ grid_utf = grid.encode('utf', resolution=self.scale, features=True)
+
+ if self.wrapper is None:
+ return SaveableResponse(json.dumps(grid_utf))
+ else:
+ return SaveableResponse(self.wrapper + '(' + json.dumps(grid_utf) + ')')
+
+ def getTypeByExtension(self, extension):
+ """ Get mime-type and format by file extension.
+
+ This only accepts "json".
+ """
+ if extension.lower() != 'json':
+ raise KnownUnknown('MapnikGrid only makes .json tiles, not "%s"' % extension)
+
+ return 'text/json', 'JSON'
+
+class SaveableResponse:
+ """ Wrapper class for JSON response that makes it behave like a PIL.Image object.
+
+ TileStache.getTile() expects to be able to save one of these to a buffer.
+ """
+ def __init__(self, content):
+ self.content = content
+
+ def save(self, out, format):
+ if format != 'JSON':
+ raise KnownUnknown('MapnikGrid only saves .json tiles, not "%s"' % format)
+
+ out.write(self.content)
View
@@ -35,10 +35,17 @@
tileset:
Required local file path to MBTiles tileset file, a SQLite 3 database file.
"""
-from sqlite3 import connect as _connect
from urlparse import urlparse, urljoin
from os.path import exists
+try:
+ from sqlite3 import connect as _connect
+except ImportError:
+ # Heroku appears to be missing standard python's
+ # sqlite3 package, so throw an ImportError later
+ def _connect(filename):
+ raise ImportError('No module named sqlite3')
+
from ModestMaps.Core import Coordinate
def create_tileset(filename, name, type, version, description, format, bounds=None):
@@ -163,6 +170,16 @@ def get_tile(filename, coord):
return mime_type, content
+def delete_tile(filename, coord):
+ """ Delete a tile by coordinate.
+ """
+ db = _connect(filename)
+ db.text_factory = bytes
+
+ tile_row = (2**coord.zoom - 1) - coord.row # Hello, Paul Ramsey.
+ q = 'DELETE FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?'
+ db.execute(q, (coord.zoom, coord.column, tile_row))
+
def put_tile(filename, coord, content):
"""
"""
@@ -242,7 +259,12 @@ def lock(self, layer, coord, format):
def unlock(self, layer, coord, format):
return
-
+
+ def remove(self, layer, coord, format):
+ """ Remove a cached tile.
+ """
+ delete_tile(self.filename, coord)
+
def read(self, layer, coord, format):
""" Return raw tile content from tileset.
"""
Oops, something went wrong.

0 comments on commit fa0184a

Please sign in to comment.