Symlink on disk for the tiles filled with 1 color

[support4net]

Hi
I think it would be a great idea to implement a symlink on the disk for tiles filled with one single color. This will save a lot of disk space and will accelerate the seeding.

[rburhum]

I think it is a great idea. Have you started experimenting with this?

[support4net]

Yes is working great on Mapproxy and is drastically reducing the disk space used. We use now the TileStache as a source for Mapproxy that is performing the sandwich better than TileStache, but the problem is the space used by TileStache for caching.

[rburhum]

Sounds great. Contributions are always welcomed. Let me know how I can help.

[support4net]

Unfortunately is impossible for us to develop this feature because we do not have Python programmers in our company :(

[duebbert]

I created an amended Disk Cache backend that does this (also compresses files with pngquant if wanted):

import gzip
import os
from os.path import dirname
from tempfile import mkstemp

import cStringIO as StringIO

from PIL import Image
from TileStache.Caches import Disk


class DiskLink(Disk, object):
    def __init__(self, path, umask=0022, dirs='safe', gzip='txt text json xml'.split(),
                 link_single_colors=False, pngquant=False):
        super(DiskLink, self).__init__(path, umask, dirs, gzip)

        self.link_single_colors = link_single_colors
        self.single_color_images_path = os.path.join(path, 'single_color_images')
        self.pngquant = pngquant

    def save(self, body, layer, coord, format):
        """ Save a cached tile.
        """
        # If we're not linking single color images, then just call the usual method
        if not self.link_single_colors:
            super(DiskLink, self).save(body, layer, coord, format)
            return

        # Check if the image is a single color image
        stream = StringIO.StringIO(body)
        im = Image.open(stream)

        fullpath = self._fullpath(layer, coord, format)

        # Get 1 color back or None if there are more than 1 color
        result = im.getcolors(1)
        if result is None:
            super(DiskLink, self).save(body, layer, coord, format)

            # Run pngquant on the temp file if activated
            if self.pngquant:
                command = "pngquant --force --ext .png %s" % fullpath
                os.system(command)

            return

        try:
            umask_old = os.umask(self.umask)
            os.makedirs(dirname(fullpath), 0777 & ~self.umask)
        except OSError, e:
            if e.errno != 17:
                raise
        finally:
            os.umask(umask_old)

        suffix = '.' + format.lower()
        suffix += self._is_compressed(format) and '.gz' or ''

        color = result[0][1]

        # Get color from palette if we are in palette mode
        if im.mode == "P":
            palette = im.getpalette()
            color = (palette[color * 3], palette[color * 3 + 1], palette[color * 3 + 2])

        single_color_image_name = ''.join('%02x' % v for v in color) + "." + format.lower()
        single_color_image_path = os.path.join(self.single_color_images_path, single_color_image_name)

        if not os.path.exists(self.single_color_images_path):
            try:
                umask_old = os.umask(self.umask)
                os.makedirs(self.single_color_images_path, 0777 & ~self.umask)
            except OSError, e:
                if e.errno != 17:
                    raise
            finally:
                os.umask(umask_old)

        if not os.path.exists(single_color_image_path):
            fh, tmp_path = mkstemp(dir=self.cachepath, suffix=suffix)

            if self._is_compressed(format):
                os.close(fh)
                tmp_file = gzip.open(tmp_path, 'w')
                tmp_file.write(body)
                tmp_file.close()
            else:
                os.write(fh, body)
                os.close(fh)

            try:
                os.rename(tmp_path, single_color_image_path)
            except OSError:
                os.unlink(single_color_image_path)
                os.rename(tmp_path, single_color_image_path)

            os.chmod(single_color_image_path, 0666 & ~self.umask)

        try:
            os.symlink(single_color_image_path, fullpath)
        except OSError, e:
            if e.errno != 17:
                raise

If you put this into a python module, then you can use it in your config like this:

            "cache": {
                "class": "CustomModule:DiskLink",
                "kwargs": {
                    "path": "/tmp/tilestache-cache",
                    "dirs": "safe",
                    "link_single_colors": true,
                    "pngquant": true
                }
            },