From 27e8008bf35b3d945a16e20b6e3fef78c0a205d1 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Fri, 25 Nov 2022 08:08:34 -0700 Subject: [PATCH 1/4] Add execution step to test scenario from #264. --- Makefile | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 55c17ae..8b976c5 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ RAM=2 # The docker-exec-default and unit-test targets run last # to make unit test results visible at the end. .PHONY: all -all: docker-exec-region docker-exec-input-file docker-exec-default unit-tests +all: docker-exec-region docker-exec-input-file docker-exec-append-w-input-file docker-exec-default unit-tests .PHONY: docker-clean docker-clean: @@ -99,6 +99,34 @@ docker-exec-input-file: build-run-docker --data-only --skip-dump --skip-nested # Make this test run faster + +.PHONE: docker-exec-append-w-input-file +docker-exec-append-w-input-file: build-run-docker + + # copy with arbitrary file name to test --input-file + docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \ + pgosm:/app/output/$(INPUT_FILE_NAME) + + # allow files created in later step to be created + docker exec -it pgosm \ + chown $(CURRENT_UID):$(CURRENT_GID) /app/output/ + # Needed for unit-tests + docker exec -it pgosm \ + chown $(CURRENT_UID):$(CURRENT_GID) /app/docker/ + + # process it, this time without providing the region but directly the filename + docker exec -it \ + -e POSTGRES_PASSWORD=mysecretpassword \ + -e POSTGRES_USER=postgres \ + -u $(CURRENT_UID):$(CURRENT_GID) \ + pgosm python3 docker/pgosm_flex.py \ + --layerset=minimal \ + --ram=$(RAM) \ + --append \ + --input-file=/app/output/$(INPUT_FILE_NAME) \ + --data-only --skip-dump --skip-nested # Make this test run faster + + .PHONE: docker-exec-region docker-exec-region: build-run-docker # copy for simulating region From e886c8ace063bf3c71195a00f1c916ab8b5fe458 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Fri, 25 Nov 2022 08:27:26 -0700 Subject: [PATCH 2/4] Reduce make processing time by switching layersets on functional tests --- Makefile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8b976c5..ec3e1a3 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,15 @@ ## ## To cleanup after you are done: ## make docker-clean +## +## The docker-exec-default target runs the "everything" layerset to +## enable testing the data processing itself. +## +## +## The other targets should define the minimal layerset to reduce processing +## times. Additional targets are testing functionality of the Docker +## processing, not the output data so loading additional data serves +## no benefit. ## ---------------------------------------------------------------------- CURRENT_UID := $(shell id -u) CURRENT_GID := $(shell id -g) @@ -20,7 +29,9 @@ RAM=2 # The docker-exec-default and unit-test targets run last # to make unit test results visible at the end. .PHONY: all -all: docker-exec-region docker-exec-input-file docker-exec-append-w-input-file docker-exec-default unit-tests +all: docker-exec-region docker-exec-input-file \ + docker-exec-append-w-input-file \ + docker-exec-default unit-tests .PHONY: docker-clean docker-clean: @@ -47,6 +58,9 @@ build-run-docker: docker-clean .PHONE: docker-exec-default docker-exec-default: build-run-docker + # This is the step used by data verification. It must run the everything + # layerset in order to test as much of the data as possible. + # copy the test data pretending it's latest to avoid downloading each time docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \ pgosm:/app/output/district-of-columbia-$(TODAY).osm.pbf @@ -93,7 +107,7 @@ docker-exec-input-file: build-run-docker -e POSTGRES_USER=postgres \ -u $(CURRENT_UID):$(CURRENT_GID) \ pgosm python3 docker/pgosm_flex.py \ - --layerset=default \ + --layerset=minimal \ --ram=$(RAM) \ --input-file=/app/output/$(INPUT_FILE_NAME) \ --data-only --skip-dump --skip-nested # Make this test run faster @@ -151,7 +165,7 @@ docker-exec-region: build-run-docker -e POSTGRES_USER=postgres \ -u $(CURRENT_UID):$(CURRENT_GID) \ pgosm python3 docker/pgosm_flex.py \ - --layerset=default \ + --layerset=minimal \ --ram=$(RAM) \ --region=$(REGION_FILE_NAME) \ --data-only --skip-dump --skip-nested # Make this test run faster From 683574fd8b1d9f48911d08d3b2ed98401f388dd1 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Fri, 25 Nov 2022 08:42:59 -0700 Subject: [PATCH 3/4] Add notes RE limits of new test step --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index ec3e1a3..13ae8f3 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,17 @@ docker-exec-input-file: build-run-docker .PHONE: docker-exec-append-w-input-file docker-exec-append-w-input-file: build-run-docker + # NOTE: This step tests --append file for an initial load. + # It does **NOT** test the actual replication process for updating data + # using append mode. Testing actual replication over time in this format + # will not be trivial. The historic file used (2021-01-13) cannot be used + # to seed a replication process, there is a time limit in upstream software + # that requires more recency to the source data. This also cannot simply + # download a file from Geofabrik, as the "latest" file will not have a diff + # to apply so also will not test the actual replication. + # + # Open a PR, Issue, discussion on https://github.com/rustprooflabs/pgosm-flex + # if you have an idea on how to implement this testing functionality. # copy with arbitrary file name to test --input-file docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \ From 49c092eadca89584b61d7dfae7df8474c33ab330 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Fri, 25 Nov 2022 08:44:44 -0700 Subject: [PATCH 4/4] Minor cleanup --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 13ae8f3..05d1cd1 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ ## enable testing the data processing itself. ## ## -## The other targets should define the minimal layerset to reduce processing -## times. Additional targets are testing functionality of the Docker -## processing, not the output data so loading additional data serves -## no benefit. +## The other targets define the "minimal" layerset to reduce processing +## times. These additional targets test functionality of the Docker +## processing, not the output data. +## Loading additional data serves no benefit. ## ---------------------------------------------------------------------- CURRENT_UID := $(shell id -u) CURRENT_GID := $(shell id -g)