Merge branch 'master' into diff_update_osm_peak_point

.env

@@ -38,3 +38,6 @@ IMPOSM_CONFIG_FILE=/usr/src/app/config/repl_config.json
 BORDERS_CLEANUP_FILE=data/borders/cleanup.pbf
 BORDERS_PBF_FILE=data/borders/filtered.pbf
 BORDERS_CSV_FILE=data/borders/lines.csv
+
+# Number of parallel processes to use when importing sql files
+MAX_PARALLEL_PSQL=5

.github/workflows/tests.yml

@@ -23,6 +23,9 @@ jobs:
           MIN_ZOOM: 0
           MAX_ZOOM: 14
         run: |
+          # For now, change the quickstart values directly in the .env file	
+          # TODO: We should probably use env vars instead	
+          sed -i 's/MAX_ZOOM=7/MAX_ZOOM=14/g' .env
           export QUIET=1
           ./quickstart.sh $area
 
@@ -39,10 +42,7 @@ jobs:
 
   performance:
     name: Evaluate performance
-    runs-on: ubuntu-latest
-    # Even though we technically don't need to wait for integrity test to finish,
-    # there is no point to run long perf test until we know the code is OK
-    needs: integrity_test
+    runs-on: self-hosted
     env:
       ## Smaller tests (runs everything in about 30 minutes)
       ## Two test areas:  equatorial-guinea  and  liechtenstein

Makefile

@@ -6,6 +6,9 @@
 SHELL         = /bin/bash
 .SHELLFLAGS   = -o pipefail -c
 
+# Make all .env variables available for make targets
+include .env
+
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit
 # Containers run as the current user rather than root (so that created files are not root-owned)
 DC_OPTS ?= --rm -u $(shell id -u):$(shell id -g)
@@ -113,8 +116,12 @@ export BORDERS_CSV_FILE ?= data/borders/$(area).lines.csv
 export MBTILES_FILE ?= $(area).mbtiles
 MBTILES_LOCAL_FILE = data/$(MBTILES_FILE)
 
-# Location of the dynamically-generated imposm config file
-export IMPOSM_CONFIG_FILE ?= data/$(area).repl.json
+ifeq ($(strip $(DIFF_MODE)),true)
+  # import-osm implementation requires IMPOSM_CONFIG_FILE to be set to a valid file
+  # For static (no-updates) import, we don't need to override the default value
+  # For the update mode, set location of the dynamically-generated area-based config file
+  export IMPOSM_CONFIG_FILE = data/$(area).repl.json
+endif
 
 # download-osm generates this file with metadata about the file
 AREA_DC_CONFIG_FILE ?= data/$(area).dc-config.yml
@@ -274,19 +281,33 @@ ifneq ($(strip $(url)),)
 	$(if $(OSM_SERVER),$(error url parameter can only be used with non-specific download target:$(newline)       make download area=$(area) url="$(url)"$(newline)))
 endif
 ifeq (,$(wildcard $(PBF_FILE)))
-	@echo "Downloading $(DOWNLOAD_AREA) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
-	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
-		if [[ "$$DIFF_MODE" == "true" ]]; then \
-			download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+ ifeq ($(strip $(DIFF_MODE)),true)
+	@echo "Downloading $(DOWNLOAD_AREA) with replication support into $(PBF_FILE) and $(IMPOSM_CONFIG_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
 				--imposm-cfg "$(IMPOSM_CONFIG_FILE)" \
-				--output "$(PBF_FILE)" ; \
-		else \
-			download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
-				--output "$(PBF_FILE)" ; \
-		fi'
+				--output "$(PBF_FILE)"
+ else
+	@echo "Downloading $(DOWNLOAD_AREA) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+				--output "$(PBF_FILE)"
+ endif
 	@echo ""
 else
+ ifeq ($(strip $(DIFF_MODE)),true)
+  ifeq (,$(wildcard $(IMPOSM_CONFIG_FILE)))
+	$(error \
+		$(newline)   Data files $(PBF_FILE) already exists, but $(IMPOSM_CONFIG_FILE) does not. \
+		$(newline)   You probably downloaded the data file before setting DIFF_MODE=true. \
+		$(newline)   You can delete the data file  $(PBF_FILE) and re-run  make download \
+		$(newline)   to re-download and generate config, or manually create  $(IMPOSM_CONFIG_FILE) \
+		$(newline)   See example    https://github.com/openmaptiles/openmaptiles-tools/blob/v5.2/bin/config/repl_config.json \
+		$(newline))
+  else
+	@echo "Data files $(PBF_FILE) and replication config $(IMPOSM_CONFIG_FILE) already exists, skipping the download."
+  endif
+ else
 	@echo "Data files $(PBF_FILE) already exists, skipping the download."
+ endif
 endif
 
 .PHONY: generate-dc-config
@@ -312,10 +333,12 @@ import-osm: all start-db-nowait
 
 .PHONY: update-osm
 update-osm: all start-db-nowait
+	@$(assert_area_is_given)
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-update'
 
 .PHONY: import-diff
 import-diff: all start-db-nowait
+	@$(assert_area_is_given)
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-diff'
 
 .PHONY: import-data

docker-compose.yml

@@ -36,10 +36,14 @@ services:
       MAKE_DC_VERSION: "2.3"
       # Allow DIFF_MODE to be overwritten from shell
       DIFF_MODE: ${DIFF_MODE}
+      # Imposm configuration file describes how to load updates when enabled
+      IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE}
       # Which files to use during import-borders processing
       BORDERS_CLEANUP_FILE: ${BORDERS_CLEANUP_FILE}
       BORDERS_PBF_FILE: ${BORDERS_PBF_FILE}
       BORDERS_CSV_FILE: ${BORDERS_CSV_FILE}
+      # Control import-sql processes
+      MAX_PARALLEL_PSQL: ${MAX_PARALLEL_PSQL}
     networks:
       - postgres_conn
     volumes:
@@ -61,7 +65,6 @@ services:
     environment:
       FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT}
       MBTILES_NAME: ${MBTILES_FILE}
-      IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE}
 
   generate-vectortiles:
     image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
@@ -74,7 +77,6 @@ services:
     environment:
       FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT}
       MBTILES_NAME: ${MBTILES_FILE}
-      IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE}
       BBOX: ${BBOX}
       MIN_ZOOM: ${MIN_ZOOM}
       MAX_ZOOM: ${MAX_ZOOM}

layers/aerodrome_label/aerodrome_label.yaml

@@ -41,7 +41,7 @@ layer:
     key_field: osm_id
     key_field_as_attribute: no
     srid: 900913
-    query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, iata, icao, ele, ele_ft FROM layer_aerodrome_label (!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
+    query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, iata, icao, ele, ele_ft FROM layer_aerodrome_label(!bbox!, z(!scale_denominator!))) AS t
 schema:
   - ./update_aerodrome_label_point.sql
   - ./layer.sql

layers/aerodrome_label/layer.sql

@@ -1,8 +1,8 @@
+
 -- etldoc: layer_aerodrome_label[shape=record fillcolor=lightpink, style="rounded,filled", label="layer_aerodrome_label | <z10_> z10+" ] ;
 
 CREATE OR REPLACE FUNCTION layer_aerodrome_label(bbox geometry,
-                                                 zoom_level integer,
-                                                 pixel_width numeric)
+                                                 zoom_level integer)
     RETURNS TABLE
             (
                 osm_id   bigint,
@@ -38,5 +38,6 @@ SELECT
 FROM osm_aerodrome_label_point
 WHERE geometry && bbox
   AND zoom_level >= 10;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/aeroway/layer.sql

@@ -56,5 +56,6 @@ FROM (
          WHERE zoom_level >= 14
      ) AS zoom_levels
 WHERE geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/boundary/boundary.sql

@@ -206,7 +206,10 @@ SELECT CASE
                THEN replace(SUBSTRING(name, POSITION(' at ' IN name) + 4), ' ', '')
            ELSE replace(replace(name, ' ', ''), 'Extentof', '')
            END;
-$$ LANGUAGE SQL IMMUTABLE;
+$$ LANGUAGE SQL IMMUTABLE
+                -- STRICT
+                PARALLEL SAFE
+                ;
 
 
 -- etldoc: ne_110m_admin_0_boundary_lines_land  -> boundary_z0
@@ -625,5 +628,6 @@ FROM (
          WHERE geometry && bbox
            AND zoom_level >= 13
      ) AS zoom_levels;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/building/building.sql

@@ -112,6 +112,9 @@ FROM (
            AND geometry && bbox
      ) AS zoom_levels
 ORDER BY render_height ASC, ST_YMin(geometry) DESC;
-$$ LANGUAGE SQL IMMUTABLE;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE
+                ;
 
 -- not handled: where a building outline covers building parts

layers/housenumber/layer.sql

@@ -18,5 +18,6 @@ SELECT
 FROM osm_housenumber_point
 WHERE zoom_level >= 14
   AND geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/landcover/landcover.sql

@@ -15,6 +15,7 @@ SELECT CASE
            %%FIELD_MAPPING: class %%
            END;
 $$ LANGUAGE SQL IMMUTABLE
+                -- STRICT
                 PARALLEL SAFE;
 
 -- etldoc: ne_110m_glaciated_areas ->  landcover_z0
@@ -185,5 +186,6 @@ FROM (
          WHERE zoom_level >= 14
            AND geometry && bbox
      ) AS zoom_levels;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/landuse/landuse.sql

@@ -214,5 +214,6 @@ FROM (
          WHERE zoom_level >= 14
      ) AS zoom_levels
 WHERE geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/mountain_peak/layer.sql

@@ -57,5 +57,6 @@ WHERE zoom_level >= 7
   AND (rank <= 5 OR zoom_level >= 14)
 ORDER BY "rank" ASC;
 
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
                 PARALLEL SAFE;
+-- TODO: Check if the above can be made STRICT -- i.e. if pixel_width could be NULL

layers/park/layer.sql

@@ -198,6 +198,7 @@ FROM (
                   FROM osm_park_polygon_gen8
                   WHERE zoom_level = 6
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon_gen7 -> layer_park:z7
@@ -214,6 +215,7 @@ FROM (
                   FROM osm_park_polygon_gen7
                   WHERE zoom_level = 7
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon_gen6 -> layer_park:z8
@@ -230,6 +232,7 @@ FROM (
                   FROM osm_park_polygon_gen6
                   WHERE zoom_level = 8
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon_gen5 -> layer_park:z9
@@ -246,6 +249,7 @@ FROM (
                   FROM osm_park_polygon_gen5
                   WHERE zoom_level = 9
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon_gen4 -> layer_park:z10
@@ -262,6 +266,7 @@ FROM (
                   FROM osm_park_polygon_gen4
                   WHERE zoom_level = 10
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon_gen3 -> layer_park:z11
@@ -278,6 +283,7 @@ FROM (
                   FROM osm_park_polygon_gen3
                   WHERE zoom_level = 11
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon_gen2 -> layer_park:z12
@@ -294,6 +300,7 @@ FROM (
                   FROM osm_park_polygon_gen2
                   WHERE zoom_level = 12
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon_gen1 -> layer_park:z13
@@ -310,6 +317,7 @@ FROM (
                   FROM osm_park_polygon_gen1
                   WHERE zoom_level = 13
                     AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
                   UNION ALL
 
                   -- etldoc: osm_park_polygon -> layer_park:z14
@@ -328,5 +336,6 @@ FROM (
                     AND geometry_point && bbox
               ) AS park_point
      ) AS park_all;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
                 PARALLEL SAFE;
+-- TODO: Check if the above can be made STRICT -- i.e. if pixel_width could be NULL

layers/place/city.sql

@@ -74,5 +74,6 @@ FROM (
             OR (zoom_level BETWEEN 11 AND 12 AND (gridrank <= 14 OR "rank" IS NOT NULL))
             OR (zoom_level >= 13)
      ) AS city_all;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/place/layer.sql

@@ -142,5 +142,6 @@ FROM (
          FROM layer_city(bbox, zoom_level, pixel_width)
          ORDER BY "rank" ASC
      ) AS place_all
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
                 PARALLEL SAFE;
+-- TODO: Check if the above can be made STRICT -- i.e. if pixel_width could be NULL

layers/poi/layer.sql

@@ -94,5 +94,6 @@ FROM (
            AND zoom_level >= 14
      ) AS poi_union
 ORDER BY "rank"
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
                 PARALLEL SAFE;
+-- TODO: Check if the above can be made STRICT -- i.e. if pixel_width could be NULL

layers/transportation/layer.sql

@@ -698,5 +698,6 @@ FROM (
      ) AS zoom_levels
 WHERE geometry && bbox
 ORDER BY z_order ASC;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/transportation_name/layer.sql

@@ -147,5 +147,6 @@ FROM (
      ) AS zoom_levels
 WHERE geometry && bbox
 ORDER BY z_order ASC;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/water/water.sql

@@ -385,5 +385,6 @@ FROM (
          WHERE zoom_level >= 14
      ) AS zoom_levels
 WHERE geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/water_name/layer.sql

@@ -74,5 +74,6 @@ WHERE geometry && bbox
         OR (zoom_level >= "rank" AND "rank" IS NOT NULL)
         OR (zoom_level >= 8)
     );
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;

layers/waterway/waterway.sql

@@ -220,5 +220,6 @@ FROM (
          WHERE zoom_level >= 14
      ) AS zoom_levels
 WHERE geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE
+$$ LANGUAGE SQL STABLE
+                -- STRICT
                 PARALLEL SAFE;