diff --git a/Makefile b/Makefile index 55c17ae..05d1cd1 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,15 @@ ## ## To cleanup after you are done: ## make docker-clean +## +## The docker-exec-default target runs the "everything" layerset to +## enable testing the data processing itself. +## +## +## The other targets define the "minimal" layerset to reduce processing +## times. These additional targets test functionality of the Docker +## processing, not the output data. +## Loading additional data serves no benefit. ## ---------------------------------------------------------------------- CURRENT_UID := $(shell id -u) CURRENT_GID := $(shell id -g) @@ -20,7 +29,9 @@ RAM=2 # The docker-exec-default and unit-test targets run last # to make unit test results visible at the end. .PHONY: all -all: docker-exec-region docker-exec-input-file docker-exec-default unit-tests +all: docker-exec-region docker-exec-input-file \ + docker-exec-append-w-input-file \ + docker-exec-default unit-tests .PHONY: docker-clean docker-clean: @@ -47,6 +58,9 @@ build-run-docker: docker-clean .PHONE: docker-exec-default docker-exec-default: build-run-docker + # This is the step used by data verification. It must run the everything + # layerset in order to test as much of the data as possible. + # copy the test data pretending it's latest to avoid downloading each time docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \ pgosm:/app/output/district-of-columbia-$(TODAY).osm.pbf @@ -93,8 +107,47 @@ docker-exec-input-file: build-run-docker -e POSTGRES_USER=postgres \ -u $(CURRENT_UID):$(CURRENT_GID) \ pgosm python3 docker/pgosm_flex.py \ - --layerset=default \ + --layerset=minimal \ + --ram=$(RAM) \ + --input-file=/app/output/$(INPUT_FILE_NAME) \ + --data-only --skip-dump --skip-nested # Make this test run faster + + + +.PHONE: docker-exec-append-w-input-file +docker-exec-append-w-input-file: build-run-docker + # NOTE: This step tests --append file for an initial load. + # It does **NOT** test the actual replication process for updating data + # using append mode. Testing actual replication over time in this format + # will not be trivial. The historic file used (2021-01-13) cannot be used + # to seed a replication process, there is a time limit in upstream software + # that requires more recency to the source data. This also cannot simply + # download a file from Geofabrik, as the "latest" file will not have a diff + # to apply so also will not test the actual replication. + # + # Open a PR, Issue, discussion on https://github.com/rustprooflabs/pgosm-flex + # if you have an idea on how to implement this testing functionality. + + # copy with arbitrary file name to test --input-file + docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \ + pgosm:/app/output/$(INPUT_FILE_NAME) + + # allow files created in later step to be created + docker exec -it pgosm \ + chown $(CURRENT_UID):$(CURRENT_GID) /app/output/ + # Needed for unit-tests + docker exec -it pgosm \ + chown $(CURRENT_UID):$(CURRENT_GID) /app/docker/ + + # process it, this time without providing the region but directly the filename + docker exec -it \ + -e POSTGRES_PASSWORD=mysecretpassword \ + -e POSTGRES_USER=postgres \ + -u $(CURRENT_UID):$(CURRENT_GID) \ + pgosm python3 docker/pgosm_flex.py \ + --layerset=minimal \ --ram=$(RAM) \ + --append \ --input-file=/app/output/$(INPUT_FILE_NAME) \ --data-only --skip-dump --skip-nested # Make this test run faster @@ -123,7 +176,7 @@ docker-exec-region: build-run-docker -e POSTGRES_USER=postgres \ -u $(CURRENT_UID):$(CURRENT_GID) \ pgosm python3 docker/pgosm_flex.py \ - --layerset=default \ + --layerset=minimal \ --ram=$(RAM) \ --region=$(REGION_FILE_NAME) \ --data-only --skip-dump --skip-nested # Make this test run faster