diff --git a/Makefile b/Makefile index cab12e3..8a085e4 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,8 @@ docker-exec-default: build-run-docker --layerset=everything \ --ram=$(RAM) \ --region=north-america/us \ - --subregion=district-of-columbia + --subregion=district-of-columbia \ + --pg-dump # pg_dump is not part of the default. Added here to ensure this usage is tested .PHONE: docker-exec-input-file @@ -110,7 +111,7 @@ docker-exec-input-file: build-run-docker --layerset=minimal \ --ram=$(RAM) \ --input-file=/app/output/$(INPUT_FILE_NAME) \ - --data-only --skip-dump --skip-nested # Make this test run faster + --data-only --skip-nested # Make this test run faster @@ -149,7 +150,7 @@ docker-exec-replication-w-input-file: build-run-docker --ram=$(RAM) \ --replication \ --input-file=/app/output/$(INPUT_FILE_NAME) \ - --data-only --skip-dump --skip-nested # Make this test run faster + --data-only --skip-nested # Make this test run faster .PHONE: docker-exec-region @@ -179,7 +180,7 @@ docker-exec-region: build-run-docker --layerset=minimal \ --ram=$(RAM) \ --region=$(REGION_FILE_NAME) \ - --data-only --skip-dump --skip-nested # Make this test run faster + --data-only --skip-nested # Make this test run faster .PHONY: unit-tests diff --git a/README.md b/README.md index b64e6d5..65e352f 100644 --- a/README.md +++ b/README.md @@ -129,14 +129,29 @@ it takes to download the 17 MB PBF file + ~ 1 minute processing. ### After processing -The `~/pgosm-data` directory has three (3) files from a single run. +The processed OpenStreetMap data is also available in the Docker container on port `5433`. +You can connect and query directly in the Docker container. + +```bash +psql -h localhost -p 5433 -d pgosm -U postgres -c "SELECT COUNT(*) FROM osm.road_line;" + +┌───────┐ +│ count │ +╞═══════╡ +│ 39865 │ +└───────┘ +``` + + +The `~/pgosm-data` directory has two (2) files from a typical single run. The PBF file and its MD5 checksum have been renamed with the date in the filename. This enables loading the file downloaded today again in the future, either with the same version of PgOSM Flex or the latest version. The `docker exec` command uses the `PGOSM_DATE` environment variable to load these historic files. -The output `.sql` is also saved in the `~/pgosm-data` directory. +If the optional `--pg-dump` option is used, the output `.sql` is also saved in +the `~/pgosm-data` directory. ```bash @@ -145,10 +160,8 @@ ls -alh ~/pgosm-data/ -rw-r--r-- 1 root root 17M Nov 2 19:57 district-of-columbia-2021-11-03.osm.pbf -rw-r--r-- 1 root root 70 Nov 2 19:59 district-of-columbia-2021-11-03.osm.pbf.md5 -rw-r--r-- 1 root root 156M Nov 3 19:10 pgosm-flex-north-america-us-district-of-columbia-default-2021-11-03.sql - ``` - This `.sql` file can be loaded into a PostGIS enabled database. The following example creates an empty `myosm` database to load the processed OpenStreetMap data into. @@ -162,23 +175,7 @@ psql -d myosm \ ``` -The processed OpenStreetMap data is also available in the Docker container on port `5433`. -You can connect and query directly in the Docker container. - -```bash -psql -h localhost -p 5433 -d pgosm -U postgres -c "SELECT COUNT(*) FROM osm.road_line;" - -┌───────┐ -│ count │ -╞═══════╡ -│ 39865 │ -└───────┘ -``` - - - -See [more in docs/DOCKER-RUN.md](docs/DOCKER-RUN.md) about other ways to customize -how PgOSM Flex runs. +> See [more in docs/DOCKER-RUN.md](docs/DOCKER-RUN.md) about other ways to customize how PgOSM Flex runs. ## Layer Sets @@ -193,12 +190,11 @@ See [docs/LAYERSETS.md](docs/LAYERSETS.md) for details. If you use QGIS to visualize OpenStreetMap, there are a few basic styles using the `public.layer_styles` table created by QGIS. +This data is loaded by default and can be excluded with `--data-only`. See [the QGIS Style README.md](https://github.com/rustprooflabs/pgosm-flex/blob/main/db/qgis-style/README.md) for more information. -This data is loaded by default and can be excluded with `--data-only`. - ## Explore data loaded diff --git a/docker/pgosm_flex.py b/docker/pgosm_flex.py index 2b64a32..1ee0a3b 100644 --- a/docker/pgosm_flex.py +++ b/docker/pgosm_flex.py @@ -29,6 +29,14 @@ Details: https://github.com/rustprooflabs/pgosm-flex/issues/275 """ +SKIP_DUMP_REMOVED_MSG = """----- ERROR ----- + +The --skip-dump option was replaced in PgOSM Flex v0.7.0 by --pg-dump. + +Details: https://github.com/rustprooflabs/pgosm-flex/issues/266 +""" + + @click.command() # Required and most common options first @click.option('--ram', required=True, @@ -61,6 +69,8 @@ @click.option('--language', default=None, envvar="PGOSM_LANGUAGE", help="Set default language in loaded OpenStreetMap data when available. e.g. 'en' or 'kn'.") +@click.option('--pg-dump', default=False, is_flag=True, + help='Uses pg_dump after processing is completed to enable easily load OpenStreetMap data into a different database') @click.option('--pgosm-date', required=False, default=helpers.get_today(), envvar="PGOSM_DATE", @@ -73,7 +83,7 @@ default='osm', help="Change the final schema name, defaults to 'osm'.") @click.option('--skip-dump', default=False, is_flag=True, - help='Skips the final pg_dump at the end. Useful for local testing when not loading into more permanent instance.') + help=SKIP_DUMP_REMOVED_MSG) @click.option('--skip-nested', default=False, is_flag=True, @@ -87,8 +97,8 @@ type=click.Choice(['append', 'create'], case_sensitive=True), help='EXPERIMENTAL - Options: create / append. Using to wrap around osm2pgsql create v. append modes, without using osm2pgsql-replication.') def run_pgosm_flex(ram, region, subregion, append, data_only, debug, - input_file, layerset, layerset_path, language, pgosm_date, - replication, schema_name, skip_dump, skip_nested, + input_file, layerset, layerset_path, language, pg_dump, + pgosm_date, replication, schema_name, skip_dump, skip_nested, srid, sp_gist, update): """Run PgOSM Flex within Docker to automate osm2pgsql flex processing. """ @@ -100,6 +110,8 @@ def run_pgosm_flex(ram, region, subregion, append, data_only, debug, # Input validation if append: sys.exit(APPEND_REMOVED_MSG) + if skip_dump: + sys.exit(SKIP_DUMP_REMOVED_MSG) if schema_name != 'osm' and replication: err_msg = 'Replication mode with custom schema name currently not supported' @@ -155,7 +167,7 @@ def run_pgosm_flex(ram, region, subregion, append, data_only, debug, dump_database(input_file=input_file, out_path=paths['out_path'], - skip_dump=skip_dump, + pg_dump=pg_dump, data_only=data_only, schema_name=schema_name) @@ -473,26 +485,26 @@ def run_post_processing(flex_path, skip_nested, import_mode): return True -def dump_database(input_file, out_path, skip_dump, data_only, schema_name): +def dump_database(input_file, out_path, pg_dump, data_only, schema_name): """Runs pg_dump when necessary to export the processed OpenStreetMap data. Parameters ----------------------- input_file : str out_path : str - skip_dump : bool + pg_dump : bool data_only : bool schema_name : str """ - if skip_dump: - logging.getLogger('pgosm-flex').info('Skipping pg_dump') - else: + if pg_dump: export_filename = get_export_filename(input_file) export_path = get_export_full_path(out_path, export_filename) db.run_pg_dump(export_path=export_path, data_only=data_only, schema_name=schema_name) + else: + logging.getLogger('pgosm-flex').info('Skipping pg_dump') def check_replication_exists():