Merge pull request #255 from sbrunner/epfl

Generate tiles on multiple dimension values, ...
camptocamp · Jul 20, 2016 · a4a4536 · a4a4536
2 parents 5e76319 + dd240c0
commit a4a4536
Show file tree

Hide file tree

Showing 48 changed files with 1,694 additions and 1,576 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -35,6 +35,8 @@ before_install:
 
 - sudo tilecloud_chain/tests/create-deploy.sh
 
+- ssh localhost echo add to know host
+
 - sudo mkdir /tmp/tests
 - sudo chmod g+s,o+w /tmp/tests
 
@@ -53,11 +55,11 @@ script:
 - vvv-validate-rst CHANGES.rst
 - python setup.py --long-description > PYPI.rst
 - vvv-validate-rst PYPI.rst
-- python setup.py nosetests --stop -a '!'nopy${TRAVIS_PYTHON_VERSION/./} -a '!'nopy`echo $TRAVIS_PYTHON_VERSION | awk -F . '{{print $1}}'`
+- python setup.py nosetests --nocapture --nologcapture --stop --attr '!'nopy`echo $TRAVIS_PYTHON_VERSION | awk -F . '{{print $1}}'`
 - if [ ${TRAVIS_PULL_REQUEST} != "false" ] ; then git diff --check ${TRAVIS_BRANCH} -- ; fi
 
 after_failure:
-- python setup.py nosetests -a '!'nopy${TRAVIS_PYTHON_VERSION/./} -a '!'nopy`echo $TRAVIS_PYTHON_VERSION | awk -F . '{{print $1}}'`
+- python setup.py nosetests --logging-filter=tilecloud,tilecloud_chain --attr '!'nopy`echo $TRAVIS_PYTHON_VERSION | awk -F . '{{print $1}}'`
 
 after_success:
 # Report coverage results to coveralls.io

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -6,3 +6,7 @@ include tilecloud_chain/OpenLayers.js
 include tilecloud_chain/OpenLayers-style.css
 include tilecloud_chain/layer-switcher-maximize.png
 include tilecloud_chain/layer-switcher-minimize.png
+include tilecloud_chain/openlayer.js
+include tilecloud_chain/openlayer.html
+include tilecloud_chain/*.jinja
+include tilecloud_chain/schema*.yaml
diff --git a/README.rst b/README.rst
@@ -889,7 +889,6 @@ Build it:
 
    git submodule update --recursive
    mkdir .build
-   virtualenv --setuptools --no-site-packages .build/venv
-   .build/venv/bin/pip install 'pip>=6' 'setuptools>=12'
-   .build/venv/bin/pip install -e .
-   .build/venv/bin/pip install -r dev-requirements.txt
+   virtualenv venv
+   venv/bin/pip install -e .
+   venv/bin/pip install -r dev-requirements.txt
diff --git a/requirements.txt b/requirements.txt
@@ -5,6 +5,7 @@ boto>=2.0
 PyYAML
 Jinja2
 pyramid
+pykwalify
 simplejson
 requests
 six
diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py
diff --git a/tilecloud_chain/amazon.py b/tilecloud_chain/amazon.py
@@ -113,7 +113,7 @@ def main():
     else:
         host = options.host
 
-    if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']:
+    if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']:  # pragma: no cover
         print("==== Sync geodata ====")
         ssh_options = ''
         if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
@@ -127,9 +127,6 @@ def main():
 
     if options.deploy_code and not options.local:
         print("==== Sync and build code ====")
-        error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
-        if error:
-            exit(1)  # pragma: no cover
 
         cmd = ['rsync', '--delete', ]
         if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
@@ -336,7 +333,7 @@ def _get_arguments(options):
         "--config", options.config,
     ]
     if options.cache is not None:
-        arguments.extend(["--destination-cache", options.cache])
+        arguments.extend(["--cache", options.cache])
     if options.layer is not None:
         arguments.extend(["--layer", options.layer])
     if options.near is not None:

diff --git a/tilecloud_chain/controller.py b/tilecloud_chain/controller.py
@@ -5,13 +5,14 @@
 import math
 import logging
 import yaml
+import pkgutil
 from six import PY3
 from six import BytesIO as StringIO
 from math import exp, log
 from copy import copy
 from argparse import ArgumentParser
 from hashlib import sha1
-from six.moves.urllib.parse import urlencode
+from six.moves.urllib.parse import urlencode, urljoin
 
 import requests
 from bottle import jinja2_template
@@ -20,7 +21,6 @@
 from tilecloud.lib.PIL_ import FORMAT_BY_CONTENT_TYPE
 
 from tilecloud_chain import TileGeneration, add_comon_options, get_tile_matrix_identifier
-from tilecloud_chain.cost import validate_calculate_cost
 
 
 logger = logging.getLogger(__name__)
@@ -42,7 +42,7 @@ def main():
         help='Generate the legend images'
     )
     parser.add_argument(
-        '--openlayers', '--generate-openlayers-test-page', default=False,
+        '--openlayers', '--generate-openlayers-testpage', default=False,
         action='store_true', dest='openlayers',
         help='Generate openlayers test page'
     )
@@ -68,11 +68,7 @@ def main():
     if options.dump_config:
         for layer in gene.config['layers'].keys():
             gene.set_layer(layer, options)
-            validate_calculate_cost(gene)
         _validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
-        gene.validate_mapcache_config()
-        gene.validate_apache_config()
-        _validate_generate_openlayers(gene)
         for grid in gene.config['grids'].values():
             if 'obj' in grid:
                 del grid['obj']
@@ -130,53 +126,29 @@ def _get(path, cache):
 
 
 def _validate_generate_wmts_capabilities(gene, cache):
-    error = False
-    error = gene.validate(cache, 'cache[%s]' % cache['name'], 'http_url', attribute_type=str, default=False) or error
-    error = gene.validate(cache, 'cache[%s]' % cache['name'], 'http_urls', attribute_type=list, default=False) or error
-    error = gene.validate(cache, 'cache[%s]' % cache['name'], 'hosts', attribute_type=list, default=False) or error
-    if not cache['http_url'] and not cache['http_urls']:  # pragma: no cover
+    if 'http_url' not in cache and 'http_urls' not in cache:  # pragma: no cover
         logger.error(
             "The attribute 'http_url' or 'http_urls' is required in the object %s." %
             ('cache[%s]' % cache['name'])
         )
-        error = True
-    if cache['http_url'] and cache['http_url'][-1] == '/':  # pragma: no cover
-        logger.error(
-            "The attribute 'http_url' shouldn't ends with a '/' in the object %s." %
-            ('cache[%s]' % cache['name'])
-        )
-        error = True
-    elif cache['http_urls']:
-        for url in cache['http_urls']:
-            if url[-1] == '/':  # pragma: no cover
-                logger.error(
-                    "The element '%s' of the attribute 'http_urls' shouldn't ends"
-                    " with a '/' in the object %s." %
-                    ('cache[%s]' % cache['name'])
-                )
-                error = True
-
-    if error:  # pragma: no cover
         exit(1)
 
 
 def _generate_wmts_capabilities(gene):
-    from tilecloud_chain.wmts_get_capabilities_template import wmts_get_capabilities_template
-
     cache = gene.caches[gene.options.cache]
     _validate_generate_wmts_capabilities(gene, cache)
     server = 'server' in gene.config
 
     base_urls = []
-    if cache['http_url']:
-        if cache['hosts']:
+    if 'http_url' in cache:
+        if 'hosts' in cache:
             cc = copy(cache)
             for host in cache['hosts']:
                 cc['host'] = host
                 base_urls.append(cache['http_url'] % cc)
         else:
             base_urls = [cache['http_url'] % cache]
-    if cache['http_urls']:
+    if 'http_urls' in cache:
         base_urls = [url % cache for url in cache['http_urls']]
 
     base_urls = [url + '/' if url[-1] != '/' else url for url in base_urls]
@@ -212,12 +184,13 @@ def _generate_wmts_capabilities(gene):
                 previous_resolution = resolution
 
     capabilities = jinja2_template(
-        wmts_get_capabilities_template,
+        pkgutil.get_data("tilecloud_chain", "wmts_get_capabilities.jinja").decode('utf-8'),
         layers=gene.layers,
         grids=gene.grids,
-        getcapabilities=base_urls[0] + (
+        getcapabilities=urljoin(base_urls[0], (
             'wmts/1.0.0/WMTSCapabilities.xml' if server
-            else cache['wmtscapabilities_file']),
+            else cache['wmtscapabilities_file']
+        )),
         base_urls=base_urls,
         base_url_postfix='wmts/' if server else '',
         get_tile_matrix_identifier=get_tile_matrix_identifier,
@@ -238,7 +211,7 @@ def _generate_legend_images(gene):
                 previous_hash = None
                 for zoom, resolution in enumerate(layer['grid_ref']['resolutions']):
                     legends = []
-                    for l in layer['layers']:
+                    for l in layer['layers'].split(','):
                         response = session.get(layer['url'] + '?' + urlencode({
                             'SERVICE': 'WMS',
                             'VERSION': '1.1.1',
@@ -253,8 +226,8 @@ def _generate_legend_images(gene):
                             legends.append(Image.open(StringIO(response.content)))
                         except:  # pragma: nocover
                             logger.warn(
-                                "Unable to read legend image for layer '%s', resolution '%i': %r" % (
-                                    layer['name'], resolution, response.content
+                                "Unable to read legend image for layer '{}'-'{}', resolution '{}': {}".format(
+                                    layer['name'], l, resolution, response.content
                                 )
                             )
                     width = max(i.size[0] for i in legends)
@@ -284,21 +257,16 @@ def _generate_legend_images(gene):
 
 
 def _generate_mapcache_config(gene):
-    from tilecloud_chain.mapcache_config_template import mapcache_config_template
-
-    if not gene.validate_mapcache_config():
-        exit(1)  # pragma: no cover
-
     for layer in gene.layers.values():
         if layer['type'] == 'wms' or 'wms_url' in layer:
-            if 'FORMAT' not in layer['params']:
-                layer['params']['FORMAT'] = layer['mime_type']
-            if 'LAYERS' not in layer['params']:
-                layer['params']['LAYERS'] = ','.join(layer['layers'])
-            if 'TRANSPARENT' not in layer['params']:
-                layer['params']['TRANSPARENT'] = 'TRUE' if layer['mime_type'] == 'image/png' else 'FALSE'
+            if 'FORMAT' not in layer.get('params', {}):
+                layer.get('params', {})['FORMAT'] = layer['mime_type']
+            if 'LAYERS' not in layer.get('params', {}):
+                layer.get('params', {})['LAYERS'] = layer['layers']
+            if 'TRANSPARENT' not in layer.get('params', {}):
+                layer.get('params', {})['TRANSPARENT'] = 'TRUE' if layer['mime_type'] == 'image/png' else 'FALSE'
     config = jinja2_template(
-        mapcache_config_template,
+        pkgutil.get_data("tilecloud_chain", "mapcache_config.jinja").decode('utf-8'),
         layers=gene.layers,
         grids=gene.grids,
         mapcache=gene.config['mapcache'],
@@ -312,9 +280,6 @@ def _generate_mapcache_config(gene):
 
 
 def _generate_apache_config(gene):
-    if not gene.validate_apache_config():
-        exit(1)  # pragma: no cover
-
     cache = gene.caches[gene.options.cache]
     use_server = 'server' in gene.config
 
@@ -325,7 +290,8 @@ def _generate_apache_config(gene):
             folder += '/'
 
         if not use_server:
-            f.write("""<Location %(location)s>
+            f.write("""
+    <Location %(location)s>
         ExpiresActive on
         ExpiresDefault "now plus %(expires)i hours"
     %(headers)s
@@ -377,9 +343,6 @@ def _generate_apache_config(gene):
                 )
 
         use_mapcache = 'mapcache' in gene.config
-        if use_mapcache:
-            if not gene.validate_mapcache_config():
-                exit(1)  # pragma: no cover
         if use_mapcache and not use_server:
             token_regex = '([a-zA-Z0-9_\-\+~\.]+)'
             f.write('\n')
@@ -422,37 +385,13 @@ def _generate_apache_config(gene):
             })
 
 
-def _validate_generate_openlayers(gene):
-    error = False
-    error = gene.validate(gene.config, 'config', 'openlayers', attribute_type=dict, default={}) or error
-    error = gene.validate(
-        gene.config['openlayers'], 'openlayers', 'srs',
-        attribute_type=str, default='EPSG:21781'
-    ) or error
-    error = gene.validate(
-        gene.config['openlayers'], 'openlayers', 'center_x',
-        attribute_type=float, default=600000
-    ) or error
-    error = gene.validate(
-        gene.config['openlayers'], 'openlayers', 'center_y',
-        attribute_type=float, default=200000
-    ) or error
-    if error:
-        exit(1)  # pragma: no cover
-
-
 def _get_resource(ressource):
     path = os.path.join(os.path.dirname(__file__), ressource)
     with open(path, 'rb') as f:
         return f.read()
 
 
 def _generate_openlayers(gene):
-    from tilecloud_chain.openlayers_html import openlayers_html
-    from tilecloud_chain.openlayers_js import openlayers_js
-
-    _validate_generate_openlayers(gene)
-
     cache = gene.caches[gene.options.cache]
 
     http_url = ''
@@ -470,7 +409,7 @@ def _generate_openlayers(gene):
         http_url += '/'
 
     js = jinja2_template(
-        openlayers_js,
+        pkgutil.get_data("tilecloud_chain", "openlayers.js").decode('utf-8'),
         srs=gene.config['openlayers']['srs'],
         center_x=gene.config['openlayers']['center_x'],
         center_y=gene.config['openlayers']['center_y'],
@@ -488,7 +427,10 @@ def _generate_openlayers(gene):
         sorted=sorted,
     )
 
-    _send(openlayers_html, 'index.html', 'text/html', cache)
+    _send(
+        pkgutil.get_data("tilecloud_chain", "openlayers.html"),
+        'index.html', 'text/html', cache
+    )
     _send(js, 'wmts.js', 'application/javascript', cache)
     _send(_get_resource('OpenLayers.js'), 'OpenLayers.js', 'application/javascript', cache)
     _send(_get_resource('OpenLayers-style.css'), 'theme/default/style.css', 'text/css', cache)

diff --git a/tilecloud_chain/copy_.py b/tilecloud_chain/copy_.py
@@ -15,13 +15,25 @@ class Copy:
     count = None
 
     def copy(self, options, gene, layer, source, dest, task_name):
+        if gene.layers[layer]['type'] == 'wms':
+            all_dimensions = gene.get_all_dimensions(gene.layers[layer])
+
+            if len(all_dimensions) == 0:
+                self._copy(options, gene, layer, source, dest, task_name)
+            else:  # pragma: no cover
+                for dimensions in all_dimensions:
+                    self._copy(options, gene, layer, source, dest, task_name, dimensions)
+        else:  # pragma: no cover
+            self._copy(options, gene, layer, source, dest, task_name)
+
+    def _copy(self, options, gene, layer, source, dest, task_name, dimensions={}):
         # disable metatiles
-        gene.layers[layer]['meta'] = False
+        del gene.layers[layer]['meta']
         count_tiles_dropped = Count()
 
         gene.set_layer(layer, options)
-        source_tilestore = gene.get_tilesstore(source)
-        dest_tilestore = gene.get_tilesstore(dest)
+        source_tilestore = gene.get_tilesstore(source, dimensions)
+        dest_tilestore = gene.get_tilesstore(dest, dimensions)
         gene.init_tilecoords()
         gene.add_geom_filter()
         gene.add_logger()
@@ -97,7 +109,10 @@ def main():
         copy = Copy()
         copy.copy(options, gene, options.layer, options.source, options.dest, 'copy')
     else:
-        for layer in gene.config['generation']['default_layers']:
+        layers = gene.config['generation']['default_layers'] \
+            if 'default_layers' in gene.config['generation'] \
+            else gene.config['layers'].keys()
+        for layer in layers:
             copy = Copy()
             copy.copy(options, gene, layer, options.source, options.dest, 'copy')
 
@@ -116,10 +131,12 @@ def process():
 
     gene = TileGeneration(options.config, options)
 
+    copy = Copy()
     if (options.layer):  # pragma: no cover
-        copy = Copy()
         copy.copy(options, gene, options.layer, options.cache, options.cache, 'process')
     else:
-        for layer in gene.config['generation']['default_layers']:
-            copy = Copy()
+        layers_name = gene.config['generation']['default_layers'] \
+            if 'default_layers' in gene.config.get('generation', {}) \
+            else gene.layers.keys()
+        for layer in layers_name:
             copy.copy(options, gene, layer, options.cache, options.cache, 'process')