Merge pull request #1844 from tilezen/zerebubuth/test-faster-sql-updates

Faster "perform SQL updates"

data/apply-planet_osm_line.sql

@@ -5,82 +5,18 @@ BEGIN
 -- planet_osm_line
 --------------------------------------------------------------------------------
 
-UPDATE planet_osm_line
-  SET mz_road_level = mz_calculate_min_zoom_roads(planet_osm_line.*)
-  WHERE mz_calculate_min_zoom_roads(planet_osm_line.*) IS NOT NULL;
-
-UPDATE planet_osm_line
-  SET mz_transit_level = mz_calculate_min_zoom_transit(planet_osm_line.*)
-  WHERE mz_calculate_min_zoom_transit(planet_osm_line.*) IS NOT NULL;
-
-UPDATE planet_osm_line
-  SET mz_earth_min_zoom = mz_calculate_min_zoom_earth(planet_osm_line.*)
-  WHERE mz_calculate_min_zoom_earth(planet_osm_line.*) IS NOT NULL;
-
-UPDATE planet_osm_line
-  SET mz_water_min_zoom = mz_calculate_min_zoom_water(planet_osm_line.*)
-  WHERE mz_calculate_min_zoom_water(planet_osm_line.*) IS NOT NULL;
-
-UPDATE planet_osm_line
-  SET mz_boundary_min_zoom = mz_calculate_min_zoom_boundaries(planet_osm_line.*)
-  WHERE mz_calculate_min_zoom_boundaries(planet_osm_line.*) IS NOT NULL;
-
-UPDATE planet_osm_line
-  SET mz_landuse_min_zoom = mz_calculate_min_zoom_landuse(planet_osm_line.*)
-  WHERE mz_calculate_min_zoom_landuse(planet_osm_line.*) IS NOT NULL;
-
-UPDATE planet_osm_line
-  SET mz_label_placement = ST_PointOnSurface(way);
-
--- only these 2 columns are relevant in lower zoom queries
-CREATE INDEX
-  planet_osm_line_geom_min_zoom_8_index
-  ON planet_osm_line USING gist(way)
-  WHERE
-    mz_landuse_min_zoom < 8 OR
-    mz_transit_level < 8;
-
--- ladder the higher zoom level indexes
-CREATE INDEX
-  planet_osm_line_geom_min_zoom_9_index
-  ON planet_osm_line USING gist(way)
-  WHERE
-    mz_earth_min_zoom < 9 OR
-    mz_landuse_min_zoom < 9 OR
-    mz_road_level < 9 OR
-    mz_transit_level < 9 OR
-    mz_water_min_zoom < 9;
-
-CREATE INDEX
-  planet_osm_line_geom_min_zoom_12_index
-  ON planet_osm_line USING gist(way)
-  WHERE
-    mz_earth_min_zoom < 12 OR
-    mz_landuse_min_zoom < 12 OR
-    mz_road_level < 12 OR
-    mz_transit_level < 12 OR
-    mz_water_min_zoom < 12;
-
-CREATE INDEX
-  planet_osm_line_geom_min_zoom_15_index
-  ON planet_osm_line USING gist(way)
-  WHERE
-    mz_earth_min_zoom < 15 OR
-    mz_landuse_min_zoom < 15 OR
-    mz_road_level < 15 OR
-    mz_transit_level < 15 OR
-    mz_water_min_zoom < 15;
-
-CREATE INDEX
-  planet_osm_line_geom_min_zoom_index
-  ON planet_osm_line USING gist(way)
+UPDATE planet_osm_line SET
+    mz_boundary_min_zoom = mz_calculate_min_zoom_boundaries(planet_osm_line.*),
+    mz_earth_min_zoom = mz_calculate_min_zoom_earth(planet_osm_line.*),
+    mz_landuse_min_zoom = mz_calculate_min_zoom_landuse(planet_osm_line.*),
+    mz_road_level = mz_calculate_min_zoom_roads(planet_osm_line.*),
+    mz_transit_level = mz_calculate_min_zoom_transit(planet_osm_line.*),
+    mz_water_min_zoom = mz_calculate_min_zoom_water(planet_osm_line.*),
+    mz_label_placement = ST_PointOnSurface(way)
   WHERE
-    mz_earth_min_zoom IS NOT NULL OR
-    mz_landuse_min_zoom IS NOT NULL OR
-    mz_road_level IS NOT NULL OR
-    mz_transit_level IS NOT NULL OR
-    mz_water_min_zoom IS NOT NULL;
+    -- NOTE: the following line isn't SQL syntax, it's replaced in the
+    -- perform-sql-updates.sh script with a range over osm_id when we're
+    -- sharding the query to make use of all CPUs, or TRUE if we're not.
+    {{SHARDING}};
 
 END $$;
-
-ANALYZE planet_osm_line;

data/apply-planet_osm_point.sql

@@ -6,76 +6,20 @@ BEGIN
 --------------------------------------------------------------------------------
 
 UPDATE planet_osm_point SET
-    mz_poi_min_zoom = mz_calculate_min_zoom_pois(planet_osm_point.*)
-    WHERE mz_calculate_min_zoom_pois(planet_osm_point.*) IS NOT NULL;
-
-UPDATE planet_osm_point
-  SET mz_earth_min_zoom = mz_calculate_min_zoom_earth(planet_osm_point.*)
-  WHERE mz_calculate_min_zoom_earth(planet_osm_point.*) IS NOT NULL;
-
-UPDATE planet_osm_point
-  SET mz_water_min_zoom = mz_calculate_min_zoom_water(planet_osm_point.*)
-  WHERE mz_calculate_min_zoom_water(planet_osm_point.*) IS NOT NULL;
-
-UPDATE planet_osm_point
-  SET mz_places_min_zoom = mz_calculate_min_zoom_places(planet_osm_point.*)
-  WHERE mz_calculate_min_zoom_places(planet_osm_point.*) IS NOT NULL;
-
-UPDATE planet_osm_point
-  SET mz_building_min_zoom = mz_calculate_min_zoom_buildings(planet_osm_point.*)
-  WHERE mz_calculate_min_zoom_buildings(planet_osm_point.*) IS NOT NULL;
-
--- ladder the point indexes
-CREATE INDEX
-  planet_osm_point_geom_min_zoom_6_index
-  ON planet_osm_point USING gist(way)
-  WHERE
-    mz_building_min_zoom < 6 OR
-    mz_earth_min_zoom < 6 OR
-    mz_places_min_zoom < 6 OR
-    mz_poi_min_zoom < 6 OR
-    mz_water_min_zoom < 6;
-
-CREATE INDEX
-  planet_osm_point_geom_min_zoom_9_index
-  ON planet_osm_point USING gist(way)
+    mz_poi_min_zoom = mz_calculate_min_zoom_pois(planet_osm_point.*),
+    mz_earth_min_zoom = mz_calculate_min_zoom_earth(planet_osm_point.*),
+    mz_water_min_zoom = mz_calculate_min_zoom_water(planet_osm_point.*),
+    mz_places_min_zoom = mz_calculate_min_zoom_places(planet_osm_point.*),
+    mz_building_min_zoom = mz_calculate_min_zoom_buildings(planet_osm_point.*)
   WHERE
-    mz_building_min_zoom < 9 OR
-    mz_earth_min_zoom < 9 OR
-    mz_places_min_zoom < 9 OR
-    mz_poi_min_zoom < 9 OR
-    mz_water_min_zoom < 9;
-
-CREATE INDEX
-  planet_osm_point_geom_min_zoom_12_index
-  ON planet_osm_point USING gist(way)
-  WHERE
-    mz_building_min_zoom < 12 OR
-    mz_earth_min_zoom < 12 OR
-    mz_places_min_zoom < 12 OR
-    mz_poi_min_zoom < 12 OR
-    mz_water_min_zoom < 12;
-
-CREATE INDEX
-  planet_osm_point_geom_min_zoom_15_index
-  ON planet_osm_point USING gist(way)
-  WHERE
-    mz_building_min_zoom < 15 OR
-    mz_earth_min_zoom < 15 OR
-    mz_places_min_zoom < 15 OR
-    mz_poi_min_zoom < 15 OR
-    mz_water_min_zoom < 15;
-
-CREATE INDEX
-  planet_osm_point_geom_min_zoom_index
-  ON planet_osm_point USING gist(way)
-  WHERE
-    mz_building_min_zoom IS NOT NULL OR
-    mz_earth_min_zoom IS NOT NULL OR
-    mz_places_min_zoom IS NOT NULL OR
-    mz_poi_min_zoom IS NOT NULL OR
-    mz_water_min_zoom IS NOT NULL;
+    (mz_calculate_min_zoom_pois(planet_osm_point.*) IS NOT NULL OR
+     mz_calculate_min_zoom_earth(planet_osm_point.*) IS NOT NULL OR
+     mz_calculate_min_zoom_water(planet_osm_point.*) IS NOT NULL OR
+     mz_calculate_min_zoom_places(planet_osm_point.*) IS NOT NULL OR
+     mz_calculate_min_zoom_buildings(planet_osm_point.*) IS NOT NULL) AND
+    -- NOTE: the following line isn't SQL syntax, it's replaced in the
+    -- perform-sql-updates.sh script with a range over osm_id when we're
+    -- sharding the query to make use of all CPUs, or TRUE if we're not.
+    {{SHARDING}};
 
 END $$;
-
-ANALYZE planet_osm_point;

data/apply-planet_osm_polygon.sql

@@ -6,113 +6,19 @@ BEGIN
 --------------------------------------------------------------------------------
 
 UPDATE planet_osm_polygon SET
-    mz_landuse_min_zoom = mz_calculate_min_zoom_landuse(planet_osm_polygon.*)
-    WHERE mz_calculate_min_zoom_landuse(planet_osm_polygon.*) IS NOT NULL;
-
-
-UPDATE planet_osm_polygon SET
-    mz_poi_min_zoom = mz_calculate_min_zoom_pois(planet_osm_polygon.*)
-    WHERE mz_calculate_min_zoom_pois(planet_osm_polygon.*) IS NOT NULL;
-
-
-UPDATE planet_osm_polygon SET
-  mz_transit_level = mz_calculate_min_zoom_transit(planet_osm_polygon.*)
-  WHERE mz_calculate_min_zoom_transit(planet_osm_polygon.*) IS NOT NULL;
-
-
-UPDATE planet_osm_polygon
-  SET mz_water_min_zoom = mz_calculate_min_zoom_water(planet_osm_polygon.*)
-  WHERE mz_calculate_min_zoom_water(planet_osm_polygon.*) IS NOT NULL;
-
-
-UPDATE planet_osm_polygon
-  SET mz_boundary_min_zoom = mz_calculate_min_zoom_boundaries(planet_osm_polygon.*)
-  WHERE mz_calculate_min_zoom_boundaries(planet_osm_polygon.*) IS NOT NULL;
-
-
-UPDATE planet_osm_polygon
-  SET mz_building_min_zoom = mz_calculate_min_zoom_buildings(planet_osm_polygon.*)
-  WHERE mz_calculate_min_zoom_buildings(planet_osm_polygon.*) IS NOT NULL;
-
-
-UPDATE planet_osm_polygon
-  SET mz_earth_min_zoom = mz_calculate_min_zoom_earth(planet_osm_polygon.*)
-  WHERE mz_calculate_min_zoom_earth(planet_osm_polygon.*) IS NOT NULL;
-
-
-UPDATE planet_osm_polygon
-  SET mz_label_placement = ST_PointOnSurface(way);
-
-
--- polygon low zoom
-CREATE INDEX
-  planet_osm_polygon_geom_min_zoom_7_index
-  ON planet_osm_polygon USING gist(way)
-  WHERE
-    mz_landuse_min_zoom < 7 OR
-    mz_poi_min_zoom < 7 OR
-    mz_transit_level < 7;
-
--- polygon zoom 7 specific query
-CREATE INDEX
-  planet_osm_polygon_geom_min_zoom_8_index
-  ON planet_osm_polygon USING gist(way)
-  WHERE
-    mz_earth_min_zoom < 8 OR
-    mz_landuse_min_zoom < 8 OR
-    mz_poi_min_zoom < 8 OR
-    mz_transit_level < 8;
-
--- ladder the rest of the polygon queries
-CREATE INDEX
-  planet_osm_polygon_geom_min_zoom_9_index
-  ON planet_osm_polygon USING gist(way)
-  WHERE
-    mz_boundary_min_zoom < 9 OR
-    mz_building_min_zoom < 9 OR
-    mz_earth_min_zoom < 9 OR
-    mz_landuse_min_zoom < 9 OR
-    mz_poi_min_zoom < 9 OR
-    mz_transit_level < 9 OR
-    mz_water_min_zoom < 9;
-
-CREATE INDEX
-  planet_osm_polygon_geom_min_zoom_12_index
-  ON planet_osm_polygon USING gist(way)
-  WHERE
-    mz_boundary_min_zoom < 12 OR
-    mz_building_min_zoom < 12 OR
-    mz_earth_min_zoom < 12 OR
-    mz_landuse_min_zoom < 12 OR
-    mz_poi_min_zoom < 12 OR
-    mz_transit_level < 12 OR
-    mz_water_min_zoom < 12;
-
-CREATE INDEX
-  planet_osm_polygon_geom_min_zoom_15_index
-  ON planet_osm_polygon USING gist(way)
-  WHERE
-    mz_boundary_min_zoom < 15 OR
-    mz_building_min_zoom < 15 OR
-    mz_earth_min_zoom < 15 OR
-    mz_landuse_min_zoom < 15 OR
-    mz_poi_min_zoom < 15 OR
-    mz_transit_level < 15 OR
-    mz_water_min_zoom < 15;
-
-CREATE INDEX
-  planet_osm_polygon_geom_min_zoom_index
-  ON planet_osm_polygon USING gist(way)
-  WHERE
-    mz_boundary_min_zoom IS NOT NULL OR
-    mz_building_min_zoom IS NOT NULL OR
-    mz_earth_min_zoom IS NOT NULL OR
-    mz_landuse_min_zoom IS NOT NULL OR
-    mz_poi_min_zoom IS NOT NULL OR
-    mz_transit_level IS NOT NULL OR
-    mz_water_min_zoom IS NOT NULL;
+   mz_boundary_min_zoom = mz_calculate_min_zoom_boundaries(planet_osm_polygon.*),
+   mz_building_min_zoom = mz_calculate_min_zoom_buildings(planet_osm_polygon.*),
+   mz_earth_min_zoom = mz_calculate_min_zoom_earth(planet_osm_polygon.*),
+   mz_landuse_min_zoom = mz_calculate_min_zoom_landuse(planet_osm_polygon.*),
+   mz_poi_min_zoom = mz_calculate_min_zoom_pois(planet_osm_polygon.*),
+   mz_transit_level = mz_calculate_min_zoom_transit(planet_osm_polygon.*),
+   mz_water_min_zoom = mz_calculate_min_zoom_water(planet_osm_polygon.*),
+   mz_label_placement = ST_PointOnSurface(way)
+ WHERE
+   -- NOTE: the following line isn't SQL syntax, it's replaced in the
+   -- perform-sql-updates.sh script with a range over osm_id when we're
+   -- sharding the query to make use of all CPUs, or TRUE if we're not.
+   {{SHARDING}};
 
 
 END $$;
-
-ANALYZE planet_osm_polygon;

data/apply-updates-non-planet-tables.sql

@@ -156,51 +156,17 @@ UPDATE ne_50m_urban_areas SET mz_label_placement = ST_PointOnSurface(the_geom);
 UPDATE ne_10m_urban_areas SET mz_label_placement = ST_PointOnSurface(the_geom);
 
 UPDATE ne_10m_urban_areas SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_10m_urban_areas_way_area_index ON ne_10m_urban_areas(way_area);
-
 UPDATE ne_50m_urban_areas SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_50m_urban_areas_way_area_index ON ne_50m_urban_areas(way_area);
-
 UPDATE ne_110m_ocean SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_110m_ocean_wayarea_index ON ne_110m_ocean(way_area);
-
 UPDATE ne_110m_lakes SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_110m_lakes_wayarea_index ON ne_110m_lakes(way_area);
-
 UPDATE ne_50m_ocean SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_50m_ocean_wayarea_index ON ne_50m_ocean(way_area);
-
 UPDATE ne_50m_lakes SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_50m_lakes_wayarea_index ON ne_50m_lakes(way_area);
-
 UPDATE ne_50m_playas SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_50m_playas_wayarea_index ON ne_50m_playas(way_area);
-
 UPDATE ne_10m_ocean SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_10m_ocean_wayarea_index ON ne_10m_ocean(way_area);
-
 UPDATE ne_10m_lakes SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_10m_lakes_wayarea_index ON ne_10m_lakes(way_area);
-
 UPDATE ne_10m_playas SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_10m_playas_wayarea_index ON ne_10m_playas(way_area);
-
 UPDATE water_polygons SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX water_polygons_wayarea_index ON water_polygons(way_area);
-
 UPDATE ne_110m_land SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_110m_land_wayarea_index ON ne_110m_land(way_area);
-
 UPDATE ne_50m_land SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_50m_land_wayarea_index ON ne_50m_land(way_area);
-
 UPDATE ne_10m_land SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX ne_10m_land_way_area_index ON ne_10m_land(way_area);
-
 UPDATE land_polygons SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
-CREATE INDEX land_polygons_wayarea_index ON land_polygons(way_area);
-
--- we look up min_label, max_label by wikidata ID.
-CREATE INDEX ne_10m_admin_0_countries_wikidata_index ON ne_10m_admin_0_countries(wikidataid);
-CREATE INDEX ne_10m_admin_0_map_units_wikidata_index ON ne_10m_admin_0_map_units(wikidataid);
-CREATE INDEX ne_10m_admin_1_states_provinces_wikidata_index ON ne_10m_admin_1_states_provinces(wikidataid);

data/indexes/land_polygons_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX land_polygons_wayarea_index ON land_polygons(way_area);

data/indexes/ne_10m_admin_0_countries_wikidata_index.sql

@@ -0,0 +1,2 @@
+-- we look up min_label, max_label by wikidata ID.
+CREATE INDEX ne_10m_admin_0_countries_wikidata_index ON ne_10m_admin_0_countries(wikidataid);

data/indexes/ne_10m_admin_0_map_units_wikidata_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_10m_admin_0_map_units_wikidata_index ON ne_10m_admin_0_map_units(wikidataid);

data/indexes/ne_10m_admin_1_states_provinces_wikidata_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_10m_admin_1_states_provinces_wikidata_index ON ne_10m_admin_1_states_provinces(wikidataid);

data/indexes/ne_10m_lakes_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_10m_lakes_wayarea_index ON ne_10m_lakes(way_area);

data/indexes/ne_10m_land_way_area_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_10m_land_way_area_index ON ne_10m_land(way_area);

data/indexes/ne_10m_ocean_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_10m_ocean_wayarea_index ON ne_10m_ocean(way_area);

data/indexes/ne_10m_playas_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_10m_playas_wayarea_index ON ne_10m_playas(way_area);

data/indexes/ne_10m_urban_areas_way_area_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_10m_urban_areas_way_area_index ON ne_10m_urban_areas(way_area);

data/indexes/ne_110m_lakes_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_110m_lakes_wayarea_index ON ne_110m_lakes(way_area);

data/indexes/ne_110m_land_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_110m_land_wayarea_index ON ne_110m_land(way_area);

data/indexes/ne_110m_ocean_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_110m_ocean_wayarea_index ON ne_110m_ocean(way_area);

data/indexes/ne_50m_lakes_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_50m_lakes_wayarea_index ON ne_50m_lakes(way_area);

data/indexes/ne_50m_land_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_50m_land_wayarea_index ON ne_50m_land(way_area);

data/indexes/ne_50m_ocean_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_50m_ocean_wayarea_index ON ne_50m_ocean(way_area);

data/indexes/ne_50m_playas_wayarea_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_50m_playas_wayarea_index ON ne_50m_playas(way_area);

data/indexes/ne_50m_urban_areas_way_area_index.sql

@@ -0,0 +1 @@
+CREATE INDEX ne_50m_urban_areas_way_area_index ON ne_50m_urban_areas(way_area);