Add Make step to test initial --append call with --input-file

Makefile

@@ -9,6 +9,15 @@
 ##
 ## To cleanup after you are done:
 ##    make docker-clean
+##
+## The docker-exec-default target runs the "everything" layerset to
+## enable testing the data processing itself.
+##
+##
+## The other targets define the "minimal" layerset to reduce processing
+## times. These additional targets test functionality of the Docker
+## processing, not the output data.
+## Loading additional data serves no benefit.
 ## ----------------------------------------------------------------------
 CURRENT_UID := $(shell id -u)
 CURRENT_GID := $(shell id -g)
@@ -20,7 +29,9 @@ RAM=2
 # The docker-exec-default and unit-test targets run last
 # to make unit test results visible at the end.
 .PHONY: all
-all: docker-exec-region docker-exec-input-file docker-exec-default unit-tests
+all: docker-exec-region docker-exec-input-file \
+	docker-exec-append-w-input-file \
+	docker-exec-default unit-tests
 
 .PHONY: docker-clean
 docker-clean:
@@ -47,6 +58,9 @@ build-run-docker: docker-clean
 
 .PHONE: docker-exec-default
 docker-exec-default: build-run-docker
+	# This is the step used by data verification. It must run the everything
+	# layerset in order to test as much of the data as possible.
+
 	# copy the test data pretending it's latest to avoid downloading each time
 	docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \
 		pgosm:/app/output/district-of-columbia-$(TODAY).osm.pbf
@@ -93,8 +107,47 @@ docker-exec-input-file: build-run-docker
 		-e POSTGRES_USER=postgres \
 		-u $(CURRENT_UID):$(CURRENT_GID) \
 		pgosm python3 docker/pgosm_flex.py  \
-		--layerset=default \
+		--layerset=minimal \
+		--ram=$(RAM) \
+		--input-file=/app/output/$(INPUT_FILE_NAME) \
+		--data-only --skip-dump --skip-nested # Make this test run faster
+
+
+
+.PHONE: docker-exec-append-w-input-file
+docker-exec-append-w-input-file: build-run-docker
+	# NOTE: This step tests --append file for an initial load.
+	# It does **NOT** test the actual replication process for updating data
+	# using append mode. Testing actual replication over time in this format
+	# will not be trivial.  The historic file used (2021-01-13) cannot be used
+	# to seed a replication process, there is a time limit in upstream software
+	# that requires more recency to the source data. This also cannot simply
+	# download a file from Geofabrik, as the "latest" file will not have a diff
+	# to apply so also will not test the actual replication.
+	#
+	# Open a PR, Issue, discussion on https://github.com/rustprooflabs/pgosm-flex
+	# if you have an idea on how to implement this testing functionality.
+
+	# copy with arbitrary file name to test --input-file
+	docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \
+		pgosm:/app/output/$(INPUT_FILE_NAME)
+
+	# allow files created in later step to be created
+	docker exec -it pgosm \
+		chown $(CURRENT_UID):$(CURRENT_GID) /app/output/
+	# Needed for unit-tests
+	docker exec -it pgosm \
+		chown $(CURRENT_UID):$(CURRENT_GID) /app/docker/
+
+	# process it, this time without providing the region but directly the filename
+	docker exec -it \
+		-e POSTGRES_PASSWORD=mysecretpassword \
+		-e POSTGRES_USER=postgres \
+		-u $(CURRENT_UID):$(CURRENT_GID) \
+		pgosm python3 docker/pgosm_flex.py  \
+		--layerset=minimal \
 		--ram=$(RAM) \
+		--append \
 		--input-file=/app/output/$(INPUT_FILE_NAME) \
 		--data-only --skip-dump --skip-nested # Make this test run faster
 
@@ -123,7 +176,7 @@ docker-exec-region: build-run-docker
 		-e POSTGRES_USER=postgres \
 		-u $(CURRENT_UID):$(CURRENT_GID) \
 		pgosm python3 docker/pgosm_flex.py  \
-		--layerset=default \
+		--layerset=minimal \
 		--ram=$(RAM) \
 		--region=$(REGION_FILE_NAME) \
 		--data-only --skip-dump --skip-nested # Make this test run faster