From 4a812710a8e8fb41d0d7c757e0dab2058971234f Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Sat, 21 Jan 2023 08:39:16 -0700 Subject: [PATCH 1/4] Starting to overhaul docs. Quite a few changes related to --replication and defaults RE pg_dump --- README.md | 67 +++++----- docs/DOCKER-RUN.md | 308 ++++++++++++++++++++++++++++----------------- 2 files changed, 231 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 65e352f..79cc9bb 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,28 @@ This project's approach is to do as much processing in the Lua styles passed along to osm2pgsql, with post-processing steps creating indexes, constraints and comments. +## Quick start + +See the [Docker Usage](#docker-usage) section below for an explanation of +these commands. + +```bash +mkdir ~/pgosm-data +export POSTGRES_USER=postgres +export POSTGRES_PASSWORD=mysecretpassword + +docker run --name pgosm -d --rm \ + -v ~/pgosm-data:/app/output \ + -v /etc/localtime:/etc/localtime:ro \ + -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ + -p 5433:5432 -d rustprooflabs/pgosm-flex + +docker exec -it \ + pgosm python3 docker/pgosm_flex.py \ + --ram=8 \ + --region=north-america/us \ + --subregion=district-of-columbia +``` ## Versions Supported @@ -57,21 +79,24 @@ package used to determine the best osm2pgsql command assumes fast SSDs. ## PgOSM via Docker -The easiest way to use PgOSM-Flex is with the -[Docker image](https://hub.docker.com/r/rustprooflabs/pgosm-flex) hosted on -Docker Hub. -The image has all the pre-requisite software installed, -handles downloading an OSM region (or subregion) -from Geofabrik, and saves an output `.sql` file with the processed data -ready to load into your database(s). -The PBF/MD5 source files are archived by date on your local storage -with the ability to easily reload them at a later date. +The PgOSM Flex +[Docker image](https://hub.docker.com/r/rustprooflabs/pgosm-flex) +is hosted on Docker Hub. +The image includes all the pre-requisite software and handles all of the options, +logic, an post-processing steps required. Features include: + +* Automatic data download from Geofabrik and validation against checksum +* Custom Flex layers built in Lua +* Mix and match layers using Layersets +* Loads to Docker-internal Postgres, or externally defined Postgres +* Supports `osm2pgsql-replication` and `osm2pgsql --append` mode +* Export processed data via `pg_dump` for loading into additional databases ### Docker usage -This section outlines a typical import using Docker to run PgOSM-Flex. -See [the full Docker instructions in docs/DOCKER-RUN.md](docs/DOCKER-RUN.md). +This section outlines a typical import using Docker to run PgOSM Flex. +See the full Docker instructions in [docs/DOCKER-RUN.md](docs/DOCKER-RUN.md). Create directory for the `.osm.pbf` file, output `.sql` file, log output, and the osm2pgsql command ran. @@ -90,7 +115,7 @@ export POSTGRES_USER=postgres export POSTGRES_PASSWORD=mysecretpassword ``` -Start the `pgosm` Docker container. At this point, PostgreSQL / PostGIS +Start the `pgosm` Docker container. At this point, Postgres / PostGIS is available on port `5433`. ```bash @@ -110,7 +135,6 @@ along with a region/subregion. * Sub-region (`district-of-columbia`) (Optional) - ```bash docker exec -it \ pgosm python3 docker/pgosm_flex.py \ @@ -128,7 +152,6 @@ it takes to download the 17 MB PBF file + ~ 1 minute processing. ### After processing - The processed OpenStreetMap data is also available in the Docker container on port `5433`. You can connect and query directly in the Docker container. @@ -152,6 +175,7 @@ to load these historic files. If the optional `--pg-dump` option is used, the output `.sql` is also saved in the `~/pgosm-data` directory. +This `.sql` file can be loaded into a PostGIS enabled database. ```bash @@ -162,21 +186,6 @@ ls -alh ~/pgosm-data/ -rw-r--r-- 1 root root 156M Nov 3 19:10 pgosm-flex-north-america-us-district-of-columbia-default-2021-11-03.sql ``` -This `.sql` file can be loaded into a PostGIS enabled database. The following example -creates an empty `myosm` database to load the processed OpenStreetMap data into. - - -```bash -psql -d postgres -c "CREATE DATABASE myosm;" -psql -d myosm -c "CREATE EXTENSION postgis;" - -psql -d myosm \ - -f ~/pgosm-data/pgosm-flex-north-america-us-district-of-columbia-default-2021-11-03.sql -``` - - -> See [more in docs/DOCKER-RUN.md](docs/DOCKER-RUN.md) about other ways to customize how PgOSM Flex runs. - ## Layer Sets diff --git a/docs/DOCKER-RUN.md b/docs/DOCKER-RUN.md index 991c411..6aced84 100644 --- a/docs/DOCKER-RUN.md +++ b/docs/DOCKER-RUN.md @@ -1,6 +1,6 @@ -# Using PgOSM Flex within Docker +# Using PgOSM Flex -This README provides details about running PgOSM-Flex using the image defined +This README provides details about running PgOSM Flex using the image defined in `Dockerfile` and the script loaded from `docker/pgosm_flex.py`. @@ -8,26 +8,33 @@ in `Dockerfile` and the script loaded from `docker/pgosm_flex.py`. Create directory for the `.osm.pbf` file and the output `.sql` file. The PBF and MD5 files downloaded from Geofabrik are stored in this directory. +This directory location is assumed in subsequent `docker run` commands. +If you change the data file path be sure to adjust `-v ~/pgosm-data:/app/output` +appropriately to link your path. ```bash mkdir ~/pgosm-data ``` -Changing this directory location requires updating the `docker run` command. -Update the line `-v ~/pgosm-data:/app/output` to link the appropriate path. -## Run Container - +## Run PgOSM Flex Container Set environment variables for the temporary Postgres connection in Docker. + +### Internal Postgres instance + +The Postgres username and password are the minimum required parameters to use +the internal Postgres database instance. + ```bash export POSTGRES_USER=postgres export POSTGRES_PASSWORD=mysecretpassword ``` + Start the `pgosm` Docker container to make PostgreSQL/PostGIS available. This command exposes Postgres inside Docker on port 5433 and establishes links to the local directory created above (`~/pgosm-data`). If your data is stored in a @@ -54,6 +61,148 @@ docker ps -a | grep pgosm > The most common reason the Docker container fails to run is not setting the `$POSTGRES_PASSWORD` env var. +Run the processing with `docker exec`. + +```bash +docker exec -it \ + pgosm python3 docker/pgosm_flex.py \ + --ram=8 \ + --region=north-america/us \ + --subregion=district-of-columbia +``` + + + +### External Postgres instance + +The PgOSM Flex Docker image can be used with Postgres instance outside the +Docker container. + +Prepare the database and permissions as described in +[POSTGRES-PERMISSIONS.md](POSTGRES-PERMISSIONS.md). + + +Set environment variables to define the connection. + +```bash +export POSTGRES_USER=your_login_role +export POSTGRES_PASSWORD=mysecretpassword +export POSTGRES_HOST=your-host-or-ip +export POSTGRES_DB=your_db_name +export POSTGRES_PORT=5432 +``` + +---- + +Note: The `POSTGRES_HOST` value is in relation to the Docker container. +Using `localhost` refers to the Docker container and will use the Postgres instance +within the Docker container, not your host running the Docker container. +Use `ip addr` to find your local host's IP address and provide that. + +---- + +Run the container with the additional environment variables. + +```bash +docker run --name pgosm -d --rm \ + -v ~/pgosm-data:/app/output \ + -v /etc/localtime:/etc/localtime:ro \ + -e POSTGRES_USER=$POSTGRES_USER \ + -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ + -e POSTGRES_HOST=$POSTGRES_HOST \ + -e POSTGRES_DB=$POSTGRES_DB \ + -e POSTGRES_PORT=$POSTGRES_PORT \ + -p 5433:5432 -d rustprooflabs/pgosm-flex +``` + +> Note: Setting `POSTGRES_HOST` to anything but `localhost` disables the drop/create database step. This means the target database must be created prior to running PgOSM Flex. + + +The `docker exec` command is the same as when using the internal Postgres instance. + +```bash +docker exec -it \ + pgosm python3 docker/pgosm_flex.py \ + --ram=8 \ + --region=north-america/us \ + --subregion=district-of-columbia +``` + + +## Use `--replication` to keep data fresh + +> The `--replication` mode seems to be stable as of 0.7.0. It was added as an experimental feature in 0.4. (originally under the --append option). + + +PgOSM Flex's `--replication` mode wraps around the `osm2pgsql-replication` package +included with `osm2pgsql`. The first time running an import with `--replication` +mode runs osm2pgsql normally, with `--slim` mode and without `--drop`. +After osm2pgsql completes, `osm2pgsql-replication init ...` is ran to setup +the DB for updates. +This mode of operation results in larger database as the intermediate osm2pgsql +tables (`--slim`) must be left in the database (no `--drop`). + + +> Important: The original `--append` option is now under `--replication`. The `--append` option was removed in PgOSM Flex 0.7.0. See [the conversation](https://github.com/rustprooflabs/pgosm-flex/issues/275#issuecomment-1340362190) for context. + + +When using replication you need to pin your process to a specific PgOSM Flex version +in the `docker run` command. When upgrading to new versions, +be sure to check the release notes for manual upgrade steps for `--replication`. +The release notes for +[PgOSM Flex 0.6.1](https://github.com/rustprooflabs/pgosm-flex/releases/tag/0.6.1) +are one example. +The notes discussed in the release notes have reference SQL scripts +under `db/data-migration` folder. + +---- + +**WARNING - Due to the ability to configure custom layersets these data-migration +scripts need manual review, and possibly manual adjustments for +your specific database and process.** + +---- + + +The other important change when using replication is to increase Postgres' `max_connections`. +See [this discussion on osm2pgsql](https://github.com/openstreetmap/osm2pgsql/discussions/1650) +for why this is necessary. + +If using the Docker-internal Postgres instance this is done with `-c max_connections=300` +in the `docker run` command. External database connections must update this +in the appropriate `postgresql.conf` file. + + +```bash +docker run --name pgosm -d --rm \ + -v ~/pgosm-data:/app/output \ + -v /etc/localtime:/etc/localtime:ro \ + -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ + -p 5433:5432 \ + -d rustprooflabs/pgosm-flex:0.7.0 \ + -c max_connections=300 +``` + + +Run the `docker exec` step with `--replication`. + +```bash +docker exec -it \ + pgosm python3 docker/pgosm_flex.py \ + --ram=8 \ + --region=north-america/us \ + --subregion=district-of-columbia \ + --pgosm-date 2022-12-30 \ + --replication +``` + +Running the above command a second time will detect that the target database +has `osm2pgsql-replication` setup and load data via the defined replication +service. + +> Note: The `--pgosm-date` parameter is ignored during subsequent imports using `--replication`. + + ## Run PgOSM-Flex @@ -188,7 +337,7 @@ docker exec -it \ --language="en" \ --srid="4326" \ --data-only \ - --skip-dump \ + --pg-dump \ --skip-nested \ --sp-gist \ --debug @@ -229,20 +378,49 @@ docker exec -it \ ``` -## Skip nested polygon calculation +## Skip nested place polygons + +The nested place polygon calculation +([explained in this post](https://blog.rustprooflabs.com/2021/01/pgosm-flex-improved-openstreetmap-places-postgis)) +adds minimal overhead to smaller regions, e.g. Colorado with a 225 MB PBF input file. +Larger regions, such as North America (12 GB PBF), +are impacted more severely as a difference in processing time. +Calculating nested place polygons for Colorado adds less than 30 seconds on an 8 minute process, +taking about 5% longer. +A larger region, such as North America, can take 33% longer adding more than +an hour and a half to the total processing time. +See [docs/PERFORMANCE.md](PERFORMANCE.md) for more details. + Use `--skip-nested` to bypass the calculation of nested admin polygons. -The nested polygon process can take considerable time on larger regions or may -be otherwise unwanted. -## Skip data export -By default the `.sql` file is created with `pg_dump` for easy loading into one or -more Postgres databases. If this file is not needed use `--skip-dump`. This saves -time and reduces disk space consumed by the process. +## Use `--pg-dump` to export data + +> The `--pg-dump` option was added in 0.7.0. Prior versions defaulted to using `pg_dump` and provided a `--skip-dump` option to override. The default now is to only use `pg_dump` when requested. See [#266](https://github.com/rustprooflabs/pgosm-flex/issues/266) for more. + + +A `.sql` file can be created using `pg_dump` as part of the processing +for easy loading into one or more external Postgres databases. +Add `--pg-dump` to the `docker exec` command to enable this feature. + +The following example +creates an empty `myosm` database to load the processed and dumped OpenStreetMap +data. + +```bash +psql -d postgres -c "CREATE DATABASE myosm;" +psql -d myosm -c "CREATE EXTENSION postgis;" + +psql -d myosm \ + -f ~/pgosm-data/pgosm-flex-north-america-us-district-of-columbia-default-2023-01-21.sql +``` + +> The above assumes a database user with `superuser` permissions is used. See [docs/POSTGRES-PERMISSIONS.md](POSTGRES-PERMISSIONS.md) for a more granular approach to permissions. -## Configure Postgres in Docker + +## Configure Postgres inside Docker Add customizations with the `-c` switch, e.g. `-c shared_buffers=1GB`, to customize Postgres' configuration at run-time in Docker. @@ -284,103 +462,3 @@ time docker exec -it \ ``` -## Use external Postgres connection - -The PgOSM Flex Docker image can be used with an external Postgres -database instead of using the in-Docker Postgres database. - -Prepare the database and permissions as described in -[POSTGRES-PERMISSIONS.md](POSTGRES-PERMISSIONS.md). - - -Set environment variables to define the connection. - -```bash -export POSTGRES_USER=your_login_role -export POSTGRES_PASSWORD=mysecretpassword -export POSTGRES_HOST=your-host-or-ip -export POSTGRES_DB=your_db_name -export POSTGRES_PORT=5432 -``` - -Run the container with the additional environment variables. - -```bash -docker run --name pgosm -d --rm \ - -v ~/pgosm-data:/app/output \ - -v /etc/localtime:/etc/localtime:ro \ - -e POSTGRES_USER=$POSTGRES_USER \ - -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ - -e POSTGRES_HOST=$POSTGRES_HOST \ - -e POSTGRES_DB=$POSTGRES_DB \ - -e POSTGRES_PORT=$POSTGRES_PORT \ - -p 5433:5432 -d rustprooflabs/pgosm-flex -``` - -> Note: Setting `POSTGRES_HOST` to anything but `localhost` disables the drop/create database step. This means the target database must be created prior to running PgOSM Flex. - - -The `docker exec` command can be used as normal. The following -example adds `--skip-dump` to remove the overhead of that final step. - - -```bash -docker exec -it \ - pgosm python3 docker/pgosm_flex.py \ - --ram=8 \ - --region=north-america/us \ - --subregion=district-of-columbia \ - --skip-dump -``` - - -## Use `--replication` for updates - -> Added as **Experimental** feature in 0.4.6. As of 0.6.2 it's nearly ready for common use. - -PgOSM Flex's `--replication` mode wraps around the `osm2pgsql-replication` package -included with `osm2pgsql`. The first time running an import with `--replication` -mode runs osm2pgsql normally, with `--slim` mode and without `--drop`. -After osm2pgsql completes, `osm2pgsql-replication init ...` is ran to setup -the DB for updates. - -> Important: The original `--append` option is now under `--replication`. The `--append` option will be removed in PgOSM Flex 0.7.0. See [the conversation](https://github.com/rustprooflabs/pgosm-flex/issues/275#issuecomment-1340362190) for context. - - - -Need to pin the PgOSM Flex version. Also need to increase Postgres' `max_connections`, see -[this discussion on osm2pgsql](https://github.com/openstreetmap/osm2pgsql/discussions/1650). - - -```bash -docker run --name pgosm -d --rm \ - -v ~/pgosm-data:/app/output \ - -v /etc/localtime:/etc/localtime:ro \ - -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ - -p 5433:5432 \ - -d rustprooflabs/pgosm-flex:0.6.3 \ - -c max_connections=300 -``` - -> Note: The instructions for `--replication` use a specific tagged version of the PgOSM Flex Docker image. Upgrading PgOSM Flex versions with replication mode is possible with manual DDL scripts. Caution and testing is strongly recommended before proceeding on production. See the release notes, along with the scripts under `pgosm-flex/db/data-migration/`. - - -Run the `docker exec` step with `--replication` and `--skip-dump`. This results in -a larger database as the intermediate osm2pgsql tables must be left -in the database. - -```bash -docker exec -it \ - pgosm python3 docker/pgosm_flex.py \ - --ram=8 \ - --region=north-america/us \ - --subregion=district-of-columbia \ - --pgosm-date 2022-12-30 \ - --replication -``` - -Running the above command a second time will detect that the target database -has `osm2pgsql-replication` setup and load data via the defined replication -service. - - From 0de0fb70e2e459b30cbdd639277c165374b2a4d7 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Sat, 21 Jan 2023 10:05:17 -0700 Subject: [PATCH 2/4] Suggest putting external connection details in private config file to source from.Additional minor updates and cleanup --- README.md | 11 ++++++----- docs/DOCKER-RUN.md | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 79cc9bb..4c85021 100644 --- a/README.md +++ b/README.md @@ -173,17 +173,18 @@ again in the future, either with the same version of PgOSM Flex or the latest ve to load these historic files. -If the optional `--pg-dump` option is used, the output `.sql` is also saved in +If `--pg-dump` option is used the output `.sql` is also saved in the `~/pgosm-data` directory. -This `.sql` file can be loaded into a PostGIS enabled database. +This `.sql` file can be loaded into any other database with PostGIS and the proper +permissions. ```bash ls -alh ~/pgosm-data/ --rw-r--r-- 1 root root 17M Nov 2 19:57 district-of-columbia-2021-11-03.osm.pbf --rw-r--r-- 1 root root 70 Nov 2 19:59 district-of-columbia-2021-11-03.osm.pbf.md5 --rw-r--r-- 1 root root 156M Nov 3 19:10 pgosm-flex-north-america-us-district-of-columbia-default-2021-11-03.sql +-rw-r--r-- 1 root root 18M Jan 21 03:45 district-of-columbia-2023-01-21.osm.pbf +-rw-r--r-- 1 root root 70 Jan 21 04:39 district-of-columbia-2023-01-21.osm.pbf.md5 +-rw-r--r-- 1 root root 163M Jan 21 16:14 north-america-us-district-of-columbia-default-2023-01-21.sql ``` diff --git a/docs/DOCKER-RUN.md b/docs/DOCKER-RUN.md index 6aced84..eda7892 100644 --- a/docs/DOCKER-RUN.md +++ b/docs/DOCKER-RUN.md @@ -82,7 +82,16 @@ Prepare the database and permissions as described in [POSTGRES-PERMISSIONS.md](POSTGRES-PERMISSIONS.md). -Set environment variables to define the connection. +Set environment variables to define the connection. Create a file with the +configuration options. + +```bash +touch ~/.pgosm-db-myproject +chmod 0700 ~/.pgosm-db-myproject +nano ~/.pgosm-db-myproject +``` + +Put in the contents. ```bash export POSTGRES_USER=your_login_role @@ -92,6 +101,12 @@ export POSTGRES_DB=your_db_name export POSTGRES_PORT=5432 ``` +Env vars can be loaded using. + +```bash +source ~/.pgosm-db-myproject +``` + ---- Note: The `POSTGRES_HOST` value is in relation to the Docker container. @@ -143,7 +158,7 @@ This mode of operation results in larger database as the intermediate osm2pgsql tables (`--slim`) must be left in the database (no `--drop`). -> Important: The original `--append` option is now under `--replication`. The `--append` option was removed in PgOSM Flex 0.7.0. See [the conversation](https://github.com/rustprooflabs/pgosm-flex/issues/275#issuecomment-1340362190) for context. +> Important: The original `--append` option is now under `--replication`. The `--append` option was removed in PgOSM Flex 0.7.0. See [#275](https://github.com/rustprooflabs/pgosm-flex/issues/275) for context. When using replication you need to pin your process to a specific PgOSM Flex version @@ -462,3 +477,18 @@ time docker exec -it \ ``` +## Monitoring the import + +You can track the query activity in the database being loaded using the +`pg_stat_activity` view from `pg_catalog`. Database connections use +`application_name = 'pgosm_flex'`. + + +```sql +SELECT * + FROM pg_catalog.pg_stat_activity + WHERE application_name = 'pgosm-flex' +; +``` + + From 217a6809c47df9b6968a5b7a822fbb2ce8b01cf0 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Sat, 21 Jan 2023 10:31:21 -0700 Subject: [PATCH 3/4] Cleanup --- docs/POSTGRES-PERMISSIONS.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/POSTGRES-PERMISSIONS.md b/docs/POSTGRES-PERMISSIONS.md index 3a6b900..1bbf62e 100644 --- a/docs/POSTGRES-PERMISSIONS.md +++ b/docs/POSTGRES-PERMISSIONS.md @@ -1,18 +1,16 @@ # Postgres permissions for PgOSM Flex -These instructions show an example of setting up a database -for use with PgOSM Flex. This should work for both -[MANUAL-STEPS-RUN.md](MANUAL-STEPS-RUN.md) and -[DOCKER-RUN.md](DOCKER-RUN.md) instructions. +These instructions show an example of setting up a Postgres database +for use with PgOSM Flex as an external database connection +described in [DOCKER-RUN.md](DOCKER-RUN.md). ## Create database and PostGIS These first steps require elevated permissions within Postgres. -`CREATE DATABASE` requires the `CREATEDB` permission, however -creating the PostGIS extension requires +`CREATE DATABASE` requires the `CREATEDB` permission. +Creating the PostGIS extension requires [Postgres superuser permissions](https://blog.rustprooflabs.com/2021/12/postgis-permissions-required). - In the target Postgres instance, create your database. ```sql @@ -57,7 +55,7 @@ These permissions should allow the full PgOSM Flex process to run. `GRANT CREATE` gives the `pgosm_flex` role far more permissions than it really needs in many cases. -Running `docker exec` with `--data-only` skips these steps and would make the `GRANT CREATE` permission unnessecary for the `pgosm_flex` role. +Running `docker exec` with `--data-only` skips these steps and would make the `GRANT CREATE` permission unnecessary for the `pgosm_flex` role. It also is often desired to not make a login role the owner of database objects. This example reduces the From b93bec2a721b433ad04496df028160ce5ef447f4 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Sun, 22 Jan 2023 13:11:31 -0700 Subject: [PATCH 4/4] Update routing doc for Docker image including cleanup for multilinestring data --- docs/ROUTING.md | 95 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/docs/ROUTING.md b/docs/ROUTING.md index b05a349..7ce4557 100644 --- a/docs/ROUTING.md +++ b/docs/ROUTING.md @@ -8,10 +8,21 @@ on running PgOSM Flex w/out Docker. ```bash -osm2pgsql --slim --drop \ - --output=flex --style=./run-road-place.lua \ - -d $PGOSM_CONN \ - ~/pgosm-data/district-of-columbia-2021-01-13.osm.pbf +cd ~/pgosm-data + +wget https://github.com/rustprooflabs/pgosm-flex/raw/main/tests/data/district-of-columbia-2021-01-13.osm.pbf +wget https://github.com/rustprooflabs/pgosm-flex/raw/main/tests/data/district-of-columbia-2021-01-13.osm.pbf.md5 +``` + +Loaded using `docker exec` command below for specific date. + +```bash +docker exec -it \ + pgosm python3 docker/pgosm_flex.py \ + --ram=8 \ + --region=north-america/us \ + --subregion=district-of-columbia \ + --pgosm-date=2021-01-13 ``` @@ -21,28 +32,63 @@ Create the `pgrouting` extension. ```sql CREATE EXTENSION IF NOT EXISTS pgrouting; +CREATE SCHEMA IF NOT EXISTS routing; ``` Prepare roads for routing using pgrouting functions. ```sql -SELECT pgr_nodeNetwork('osm.road_line', .1, 'osm_id', 'geom'); -SELECT pgr_createTopology('osm.road_line_noded', 0.1, 'geom'); -SELECT pgr_analyzeGraph('osm.road_line_noded', 0.1, 'geom'); +CREATE TABLE routing.road_line AS +WITH a AS ( +SELECT osm_id, osm_type, maxspeed, oneway, layer, + route_foot, route_cycle, route_motor, access, + ST_LineMerge(geom) AS geom + FROM osm.road_line +), extra_cleanup AS ( +SELECT osm_id, osm_type, maxspeed, oneway, layer, + route_foot, route_cycle, route_motor, access, + (ST_Dump(geom)).geom AS geom + FROM a + WHERE ST_GeometryType(geom) = 'ST_MultiLineString' +), combined AS ( +SELECT osm_id, osm_type, maxspeed, oneway, layer, + route_foot, route_cycle, route_motor, access, + geom + FROM a + WHERE ST_GeometryType(geom) != 'ST_MultiLineString' +UNION +SELECT osm_id, osm_type, maxspeed, oneway, layer, + route_foot, route_cycle, route_motor, access, + geom + FROM extra_cleanup + WHERE ST_GeometryType(geom) != 'ST_MultiLineString' +) +SELECT ROW_NUMBER() OVER (ORDER BY geom) AS id, * + FROM combined + ORDER BY geom +; + +``` + + +```sql +SELECT pgr_nodeNetwork('routing.road_line', .1, 'id', 'geom'); +SELECT pgr_createTopology('routing.road_line_noded', 0.1, 'geom'); +SELECT pgr_analyzeGraph('routing.road_line_noded', 0.1, 'geom'); ``` These commands create two (2) new tables usable by pgrouting. -* `osm.road_line_noded` -* `osm.road_line_noded_vertices_pgr` +* `routing.road_line_noded` +* `routing.road_line_noded_vertices_pgr` -Add simple `cost_length` column to the `osm.road_line_noded` table +Add simple `cost_length` column to the `routing.road_line_noded` table as a generated column to use for routing costs. ```sql -ALTER TABLE osm.road_line_noded +ALTER TABLE routing.road_line_noded ADD cost_length DOUBLE PRECISION NOT NULL GENERATED ALWAYS AS (ST_Length(geom)) STORED; @@ -58,6 +104,7 @@ of road that is tagged as `highway=residential` and `access=private`. This was picked to illustrate how the calculated access control columns, `route_motor`, `route_cycle` and `route_foot`, can influence route selection. +> Note: The vertex IDs in my test database will not necessary match the vertex IDs in your database! ```bash Name |Value | @@ -88,11 +135,11 @@ SELECT d.*, n.the_geom AS node_geom, e.geom AS edge_geom FROM pgr_dijkstra( 'SELECT id, source, target, cost_length AS cost, geom - FROM osm.road_line_noded', + FROM routing.road_line_noded', 11322, 7653, directed := False ) d - INNER JOIN osm.road_line_noded_vertices_pgr n ON d.node = n.id - LEFT JOIN osm.road_line_noded e ON d.edge = e.id + INNER JOIN routing.road_line_noded_vertices_pgr n ON d.node = n.id + LEFT JOIN routing.road_line_noded e ON d.edge = e.id ; ``` @@ -112,14 +159,14 @@ SELECT d.*, n.the_geom AS node_geom, e.geom AS edge_geom FROM pgr_dijkstra( 'SELECT n.id, n.source, n.target, n.cost_length AS cost, n.geom - FROM osm.road_line_noded n + FROM routing.road_line_noded n INNER JOIN osm.road_line r ON n.old_id = r.osm_id AND route_motor', 11322, 7653, directed := False ) d - INNER JOIN osm.road_line_noded_vertices_pgr n ON d.node = n.id - LEFT JOIN osm.road_line_noded e ON d.edge = e.id + INNER JOIN routing.road_line_noded_vertices_pgr n ON d.node = n.id + LEFT JOIN routing.road_line_noded e ON d.edge = e.id ; ``` @@ -142,11 +189,11 @@ Valid values are: Assuming a noded roads table routing table, bring over the `oneway` detail ```sql -ALTER TABLE osm.road_line_noded +ALTER TABLE routing.road_line_noded ADD oneway INT2 NULL ; -UPDATE osm.road_line_noded rn +UPDATE routing.road_line_noded rn SET oneway = r.oneway FROM osm.road_line r WHERE rn.old_id = r.osm_id AND rn.oneway IS NULL @@ -157,7 +204,7 @@ Calculate forward cost. ```sql -- Cost with oneway considerations -ALTER TABLE osm.road_line_noded +ALTER TABLE routing.road_line_noded ADD cost_length NUMERIC GENERATED ALWAYS AS ( CASE WHEN oneway IN (0, 1) @@ -174,7 +221,7 @@ Reverse cost. ```sql -- Reverse cost with oneway considerations -ALTER TABLE osm.road_line_noded +ALTER TABLE routing.road_line_noded ADD cost_length_reverse NUMERIC GENERATED ALWAYS AS ( CASE WHEN oneway IN (0, -1) @@ -195,7 +242,7 @@ SELECT d.*, n.the_geom AS node_geom, e.geom AS edge_geom 'SELECT n.id, n.source, n.target, n.cost_length AS cost, n.cost_length_reverse AS reverse_cost, n.geom - FROM osm.road_line_noded n + FROM routing.road_line_noded n INNER JOIN osm.road_line r ON n.old_id = r.osm_id AND route_motor @@ -204,7 +251,7 @@ SELECT d.*, n.the_geom AS node_geom, e.geom AS edge_geom 39877, 8227, directed := True ) d - INNER JOIN osm.road_line_noded_vertices_pgr n ON d.node = n.id - LEFT JOIN osm.road_line_noded e ON d.edge = e.id + INNER JOIN routing.road_line_noded_vertices_pgr n ON d.node = n.id + LEFT JOIN routing.road_line_noded e ON d.edge = e.id ; ```