Merge pull request #17 from openmaptiles/datadoc

Documenting :  graphviz based  ETL graph

.travis.yml

@@ -0,0 +1,29 @@
+sudo: required
+
+dist: precise
+
+language: bash
+
+services:
+  - docker
+
+before_install:
+  - sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-precise main" > /etc/apt/sources.list.d/docker.list'
+  - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+  - sudo apt-get clean
+  - sudo apt-get update
+  - sudo apt-key update
+  - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.12.0-0~precise
+  - sudo rm -f /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/1.8.0/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
+  - sudo service docker restart
+  - docker-compose -v
+  - docker -v
+
+script:
+  - sudo service docker restart
+  - docker-compose config
+  - sudo ./quickstart.sh
+

docker-compose-test-override.yml

@@ -0,0 +1,8 @@
+version: "2"
+
+services:
+  generate-vectortiles:
+    environment:
+      BBOX: "8.25,46.97,9.58,47.52"
+      MIN_ZOOM: "0"
+      MAX_ZOOM: "7"

layers/boundary/boundary.sql

@@ -1,8 +1,15 @@
+
+
+-- etldoc: ne_110m_admin_0_boundary_lines_land  -> boundary_z0
+
 CREATE OR REPLACE VIEW boundary_z0 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_110m_admin_0_boundary_lines_land
 );
 
+-- etldoc: ne_50m_admin_0_boundary_lines_land  -> boundary_z1
+-- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z1
+
 CREATE OR REPLACE VIEW boundary_z1 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_50m_admin_0_boundary_lines_land
@@ -12,6 +19,10 @@ CREATE OR REPLACE VIEW boundary_z1 AS (
     WHERE scalerank <= 2
 );
 
+
+-- etldoc: ne_50m_admin_0_boundary_lines_land -> boundary_z3
+-- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z3
+
 CREATE OR REPLACE VIEW boundary_z3 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_50m_admin_0_boundary_lines_land
@@ -20,6 +31,10 @@ CREATE OR REPLACE VIEW boundary_z3 AS (
     FROM ne_50m_admin_1_states_provinces_lines
 );
 
+
+-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z4
+-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z4
+
 CREATE OR REPLACE VIEW boundary_z4 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_10m_admin_0_boundary_lines_land
@@ -29,6 +44,9 @@ CREATE OR REPLACE VIEW boundary_z4 AS (
     WHERE scalerank <= 3 AND featurecla = 'Adm-1 boundary'
 );
 
+-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z5
+-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z5
+
 CREATE OR REPLACE VIEW boundary_z5 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_10m_admin_0_boundary_lines_land
@@ -38,6 +56,8 @@ CREATE OR REPLACE VIEW boundary_z5 AS (
     WHERE scalerank <= 7 AND featurecla = 'Adm-1 boundary'
 );
 
+-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z7
+-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z7
 CREATE OR REPLACE VIEW boundary_z7 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_10m_admin_0_boundary_lines_land
@@ -48,60 +68,80 @@ CREATE OR REPLACE VIEW boundary_z7 AS (
 
 );
 
+-- etldoc: osm_boundary_linestring_gen5 -> boundary_z8
 CREATE OR REPLACE VIEW boundary_z8 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen5
     WHERE admin_level <= 4 AND ST_Length(geometry) > 1000
 );
 
+-- etldoc: osm_boundary_linestring_gen4 -> boundary_z9
 CREATE OR REPLACE VIEW boundary_z9 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen4
     WHERE admin_level <= 6
 );
 
+-- etldoc: osm_boundary_linestring_gen3 -> boundary_z10
 CREATE OR REPLACE VIEW boundary_z10 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen3
     WHERE admin_level <= 6
 );
 
+-- etldoc: osm_boundary_linestring_gen2 -> boundary_z11
 CREATE OR REPLACE VIEW boundary_z11 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen2
     WHERE admin_level <= 8
 );
 
+-- etldoc: osm_boundary_linestring_gen1 -> boundary_z12
 CREATE OR REPLACE VIEW boundary_z12 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen1
 );
 
+-- etldoc: layer_boundary[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="<sql> layer_boundary |<z0> z0 |<z1_2> z1_2 | <z3> z3 | <z4> z4 |<z5_6> z5_6 | <z7> z7 | <z8> z8 | <z9> z9    |<z10> z10 |<z11> z11 |<z12> z12|<z13_> z13_ " ] 
+
 CREATE OR REPLACE FUNCTION layer_boundary (bbox geometry, zoom_level int)
 RETURNS TABLE(geometry geometry, admin_level int) AS $$
     SELECT geom, admin_level FROM (
+        -- etldoc: boundary_z0 ->  layer_boundary:z0
         SELECT * FROM boundary_z0 WHERE geom && bbox AND zoom_level = 0
         UNION ALL
+        -- etldoc: boundary_z1 ->  layer_boundary:z1_2
         SELECT * FROM boundary_z1 WHERE geom && bbox AND zoom_level BETWEEN 1 AND 2
         UNION ALL
+        -- etldoc: boundary_z3 ->  layer_boundary:z3
         SELECT * FROM boundary_z3 WHERE geom && bbox AND zoom_level = 3
         UNION ALL
+        -- etldoc: boundary_z4 ->  layer_boundary:z4
         SELECT * FROM boundary_z4 WHERE geom && bbox AND zoom_level = 4
         UNION ALL
+        -- etldoc: boundary_z5 ->  layer_boundary:z5_6
         SELECT * FROM boundary_z5 WHERE geom && bbox AND zoom_level BETWEEN 5 AND 6
         UNION ALL
+        -- etldoc: boundary_z7 ->  layer_boundary:z7
         SELECT * FROM boundary_z7 WHERE geom && bbox AND zoom_level = 7
         UNION ALL
+        -- etldoc: boundary_z8 ->  layer_boundary:z8
         SELECT * FROM boundary_z8 WHERE geom && bbox AND zoom_level = 8
         UNION ALL
+        -- etldoc: boundary_z9 ->  layer_boundary:z9
         SELECT * FROM boundary_z9 WHERE geom && bbox AND zoom_level = 9
         UNION ALL
+        -- etldoc: boundary_z10 ->  layer_boundary:z10
         SELECT * FROM boundary_z10 WHERE geom && bbox AND zoom_level = 10
         UNION ALL
+        -- etldoc: boundary_z11 ->  layer_boundary:z11
         SELECT * FROM boundary_z11 WHERE geom && bbox AND zoom_level = 11
         UNION ALL
+        -- etldoc: boundary_z12 ->  layer_boundary:z12
         SELECT * FROM boundary_z12 WHERE geom && bbox AND zoom_level = 12
         UNION ALL
+        -- etldoc: boundary_z12 -> layer_boundary:z13_
         SELECT * FROM boundary_z12 WHERE geom && bbox AND zoom_level >= 13
     ) AS zoom_levels;
 $$ LANGUAGE SQL IMMUTABLE;

layers/boundary/mapping.yaml

@@ -1,20 +1,32 @@
 generalized_tables:
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen5 
   boundary_linestring_gen5:
     source: boundary_linestring
     tolerance: 420.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen4
   boundary_linestring_gen4:
     source: boundary_linestring
     tolerance: 320.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen3    
   boundary_linestring_gen3:
     source: boundary_linestring
     tolerance: 150.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen2    
   boundary_linestring_gen2:
     source: boundary_linestring
     tolerance: 100.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen1    
   boundary_linestring_gen1:
     source: boundary_linestring
     tolerance: 50.0
+
 tables:
+  # etldoc: imposm3 -> osm_boundary_linestring
   boundary_linestring:
     fields:
     - name: osm_id

layers/building/building.sql

@@ -1,11 +1,20 @@
+
+
+
+-- etldoc: layer_building[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_building | <z13> z13 | <z14_> z14_ " ] ;
+
 CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int)
 RETURNS TABLE(geom geometry, osm_id bigint, render_height numeric) AS $$
     SELECT geometry, osm_id,
     least(greatest(3, COALESCE(height, levels*3.66,5)),400)^.7::int AS render_height
     FROM (
+
+        -- etldoc: osm_building_polygon_gen1 -> layer_building:z13
         SELECT osm_id, geometry, height, levels FROM osm_building_polygon_gen1
         WHERE zoom_level = 13 AND geometry && bbox AND area > 1400
         UNION ALL
+        -- etldoc: osm_building_polygon -> layer_building:z14_
         SELECT osm_id, geometry, height, levels FROM osm_building_polygon
         WHERE zoom_level >= 14 AND geometry && bbox
     ) AS zoom_levels

layers/building/mapping.yaml

@@ -1,9 +1,14 @@
 generalized_tables:
+
+  # etldoc: imposm3 -> osm_building_polygon_gen1
   building_polygon_gen1:
     source: building_polygon
     sql_filter: area>1400.0
     tolerance: 10.0
+
 tables:
+
+  # etldoc: imposm3 -> osm_building_polygon
   building_polygon:
     fields:
     - name: osm_id

layers/highway/highway.sql

@@ -1,56 +1,80 @@
+
+
 CREATE OR REPLACE FUNCTION highway_is_link(highway TEXT) RETURNS BOOLEAN AS $$
     SELECT highway LIKE '%_link';
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 
+
+-- etldoc: layer_highway[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="<sql> layer_highway |<z4z7> z4-z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14_" ] ;
 CREATE OR REPLACE FUNCTION layer_highway(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass text, properties highway_properties) AS $$
     SELECT
 		osm_id, geometry,
 		to_highway_class(highway) AS class, highway AS subclass,
 		to_highway_properties(is_bridge, is_tunnel, is_ford, is_ramp, is_oneway) AS properties
 	FROM (
-		SELECT
+
+		-- etldoc: ne_10m_global_roads ->  layer_highway:z4z7
+		SELECT 
 			NULL::bigint AS osm_id, geometry, highway,
 			FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway,
 			0 AS z_order
 		FROM ne_10m_global_roads
 		WHERE zoom_level BETWEEN 4 AND 7 AND scalerank <= 1 + zoom_level
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen4  ->  layer_highway:z8
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen4
 		WHERE zoom_level = 8
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen3  ->  layer_highway:z9
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen3
 		WHERE zoom_level = 9
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen2  ->  layer_highway:z10
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen2
 		WHERE zoom_level = 10
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen1  ->  layer_highway:z11
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen1
 		WHERE zoom_level = 11
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z12
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level = 12
             AND (to_highway_class(highway) < 'minor_road'::highway_class OR highway IN ('unclassified', 'residential'))
             AND NOT highway_is_link(highway)
             AND NOT is_area
         UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z13
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level = 13
             AND to_highway_class(highway) < 'path'::highway_class
             AND NOT is_area
         UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z14_
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level >= 14 AND NOT is_area
         UNION ALL
+
         -- NOTE: We limit the selection of polys because we need to be careful to net get false positives here because
         -- it is possible that closed linestrings appear both as highway linestrings and as polygon
+        -- etldoc: osm_highway_polygon          ->  layer_highway:z13
+        -- etldoc: osm_highway_polygon          ->  layer_highway:z14_
 		SELECT osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, z_order
         FROM osm_highway_polygon
         -- We do not want underground pedestrian areas for now

layers/highway/mapping.yaml

@@ -1,21 +1,32 @@
+
 generalized_tables:
+
+# etldoc: imposm3 -> osm_highway_linestring_gen4
   highway_linestring_gen4:
     source: highway_linestring_gen3
     sql_filter: highway IN ('motorway','trunk') AND NOT is_area
     tolerance: 200.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen3    
   highway_linestring_gen3:
     source: highway_linestring_gen2
     sql_filter: highway IN ('motorway','trunk', 'primary') AND NOT is_area
     tolerance: 120.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen2    
   highway_linestring_gen2:
     source: highway_linestring_gen1
     sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary') AND NOT is_area
     tolerance: 50.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen1    
   highway_linestring_gen1:
     source: highway_linestring
     sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary', 'tertiary') AND NOT is_area
     tolerance: 20.0
+
 tables:
+# etldoc: imposm3 -> osm_highway_linestring
   highway_linestring:
     type: linestring
     fields:
@@ -85,6 +96,8 @@ tables:
       - corridor
       - crossing
       - pedestrian
+
+# etldoc: imposm3 -> osm_highway_polygon     
   highway_polygon:
     type: polygon
     fields:

layers/highway/ne_global_roads.sql

@@ -1,3 +1,5 @@
+
+
 CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$
   SELECT CASE type
     WHEN 'Major Highway' THEN 'motorway'
@@ -7,13 +9,19 @@ CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$
   END;
 $$ LANGUAGE SQL IMMUTABLE;
 
+
+-- etldoc: ne_global_roads_sql -> ne_10m_global_roads  ;
 CREATE TABLE IF NOT EXISTS ne_10m_global_roads AS (
+
+    -- etldoc:  ne_10m_roads -> ne_10m_global_roads
     SELECT geom AS geometry, scalerank, ne_highway(type) AS highway
     FROM ne_10m_roads
     WHERE continent <> 'North America'
       AND featurecla = 'Road'
       AND type IN ('Major Highway', 'Secondary Highway', 'Road')
     UNION ALL
+
+    -- etldoc: ne_10m_roads_north_america ->  ne_10m_global_roads  
     SELECT geom AS geometry, scalerank, ne_highway(type) AS highway
     FROM ne_10m_roads_north_america
     WHERE type IN ('Major Highway', 'Secondary Highway', 'Road')