added config upate check (#133)

Dockerfile

@@ -17,11 +17,11 @@ FROM ubuntu:23.04 as runner_base
 MAINTAINER Nils Nolde <nils@gis-ops.com>
 
 RUN apt-get update > /dev/null && \
-    export DEBIAN_FRONTEND=noninteractive && \
-    apt-get install -y libluajit-5.1-2 \
-      libzmq5 libczmq4 spatialite-bin libprotobuf-lite32 sudo locales \
-      libsqlite3-0 libsqlite3-mod-spatialite libcurl4 \
-      python3.11-minimal python3-distutils curl unzip moreutils jq spatialite-bin python-is-python3 > /dev/null
+  export DEBIAN_FRONTEND=noninteractive && \
+  apt-get install -y libluajit-5.1-2 \
+  libzmq5 libczmq4 spatialite-bin libprotobuf-lite32 sudo locales \
+  libsqlite3-0 libsqlite3-mod-spatialite libcurl4 \
+  python3.11-minimal python3-distutils curl unzip moreutils jq spatialite-bin python-is-python3 > /dev/null
 
 COPY --from=builder /usr/local /usr/local
 COPY --from=builder /usr/lib/python3/dist-packages/valhalla/* /usr/lib/python3/dist-packages/valhalla/
@@ -31,6 +31,7 @@ ENV LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}"
 ENV use_tiles_ignore_pbf=True
 ENV build_tar=True
 ENV serve_tiles=True
+ENV update_existing_config=True
 
 # what this does:
 # if the docker user specified a UID/GID (other than 0, would be a ludicrous instruction anyways) in the image build, we will use that to create the valhalla linux user in the image. that ensures that the docker user can edit the created files on the host without sudo and with 664/775 permissions, so that users of that group can also write. the default is to give the valhalla user passwordless sudo. that also means that all commands creating files in the entrypoint script need to be executed with sudo when built with defaults..
@@ -53,10 +54,10 @@ WORKDIR /custom_files
 
 # Smoke tests
 RUN python -c "import valhalla,sys; print (sys.version, valhalla)" \
-    && valhalla_build_config | jq type \
-    && cat /usr/local/src/valhalla_version \
-    && valhalla_build_tiles -v \
-    && ls -la /usr/local/bin/valhalla*
+  && valhalla_build_config | jq type \
+  && cat /usr/local/src/valhalla_version \
+  && valhalla_build_tiles -v \
+  && ls -la /usr/local/bin/valhalla*
 
 ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
 ENV LD_LIBRARY_PATH /usr/local/lib:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib32:/usr/lib32

README.md

@@ -17,12 +17,12 @@ This image aims at being user-friendly and most efficient with your time and res
 
 ## Features
 
--   Easily switch graphs by mapping different volumes to containers.
--   Stores all relevant data (tiles, config, admin & timezone DBs, elevation) in the mapped volume.
--   Load and build from **multiple URLs** pointing to valid pbf files.
--   Load local data through volume mapping.
--   **Supports auto rebuild** on OSM file changes through hash mapping.
-- 	**new**: supports advanced user management to avoid sudo access to host-shared folders and files, see [notes on user management](#notes-on-user-management)
+- Easily switch graphs by mapping different volumes to containers.
+- Stores all relevant data (tiles, config, admin & timezone DBs, elevation) in the mapped volume.
+- Load and build from **multiple URLs** pointing to valid pbf files.
+- Load local data through volume mapping.
+- **Supports auto rebuild** on OSM file changes through hash mapping.
+-     **new**: supports advanced user management to avoid sudo access to host-shared folders and files, see [notes on user management](#notes-on-user-management)
 
 ## Dockerhub/Github Packages
 
@@ -67,6 +67,7 @@ This image respects the following custom environment variables to be passed duri
 - `serve_tiles`: `True` starts the valhalla service. Default `True`.
 - `tileset_name`: The name of the resulting graph on disk. Very useful in case you want to build multiple datasets in the same directory. Default `valhalla_tiles`.
 - `traffic_name`: The name of the traffic.tar. Again, useful for serving mulitple traffic archives from the same directory. If empty, i.e. "", then no traffic archive will be built. Default `traffic.tar`.
+- `update_existing_config`: `True` updates missing keys in existing valhalla.json. Useful for updating stale config files to include newly introduced config parameters. Default `True`.
 
 ## Container recipes
 

docker-compose.yml.template

@@ -26,6 +26,7 @@ services:
       - build_transit=False  # build transit, needs existing GTFS directories mapped to /gtfs_feeds
       - build_tar=True  # build an indexed tar file from the tile_dir for faster graph loading times
       - force_rebuild=False  # forces a rebuild of the routing tiles with "True"
+      - update_existing_config=True  # if there are new config entries in the default config, add them to the existing config  
       # - path_extension=graphs  # this path will be internally appended to /custom_files; no leading or trailing path separator!
     # adapt to the expected build time
     # healthcheck: 

scripts/configure_valhalla.sh

@@ -116,6 +116,32 @@ files=$(echo $files | xargs)
 
 # be careful how to write the config (mostly for restart scenarios where env vars are true all of a sudden)
 if test -f "${CONFIG_FILE}"; then
+
+  if [[ "${update_existing_config}" == "True" ]]; then 
+    echo "INFO: Found existing valhalla.json. Updating possibly missing entries."
+
+    # create temporary default config
+    valhalla_build_config > ${TMP_CONFIG_FILE}  || exit 1
+
+    # for each path in the temp config (excluding array indices)
+    jq -r 'paths | select(map(type) | index("number") | not ) | "." + join(".")' ${TMP_CONFIG_FILE} | while read key ; do
+
+      # if the key path does not exist in the existing config 
+      jq -e "${key} | if type == \"null\" then false else true end" ${CONFIG_FILE} >/dev/null
+      if [ $? -eq 1 ]; then 
+
+          # get its value from the temp config
+          newval=$(jq "${key}" ${TMP_CONFIG_FILE})
+          echo "INFO: copied new config entry ${key}=${newval} into existing config."
+
+          # and set it on the new one
+          jq --argjson d "${newval}" "${key} = \$d" "${CONFIG_FILE}"| sponge "${CONFIG_FILE}"
+      fi  
+    done 
+
+    rm ${TMP_CONFIG_FILE}
+  fi
+
   jq --arg d "${TILE_DIR}" '.mjolnir.tile_dir = $d' "${CONFIG_FILE}"| sponge "${CONFIG_FILE}"
   jq --arg d "${TILE_TAR}" '.mjolnir.tile_extract = $d' "${CONFIG_FILE}"| sponge "${CONFIG_FILE}"
   jq --arg d "${ADMIN_DB}" '.mjolnir.admin = $d' "${CONFIG_FILE}"| sponge "${CONFIG_FILE}"

scripts/helpers.sh

@@ -11,6 +11,9 @@ if ! test -d "${CUSTOM_FILES}"; then
   mkdir "${CUSTOM_FILES}"
 fi
 
+# temp valhalla config location
+TMP_CONFIG_FILE="${CUSTOM_FILES}/valhalla_default.json"
+
 CONFIG_FILE="${CUSTOM_FILES}/valhalla.json"
 TILE_DIR="${CUSTOM_FILES}/${tileset_name:-valhalla_tiles}"
 TILE_TAR="${CUSTOM_FILES}/${tileset_name:-valhalla_tiles}.tar"

tests/test.sh

@@ -100,6 +100,51 @@ if [[ $(last_mod_tiles $tileset_name) != $mod_date_tiles ]]; then
   exit 1
 fi
 
+### Remove a config entry and check if it is added back ###
+echo "#### Update config test ####"
+cp "${custom_file_folder}/valhalla.json" "${custom_file_folder}/valhalla_base.json"
+jq 'del(.meili.default.beta)' "${custom_file_folder}/valhalla.json" | sponge "${custom_file_folder}/valhalla.json"
+docker restart valhalla_full
+wait_for_docker
+
+# should have been added back
+jq -e '.meili.default.beta' "${custom_file_folder}/valhalla.json" >/dev/null
+
+if [[ $? -eq 1 ]]; then
+  echo "valhalla.json should have been updated but wasn't."
+  exit 1
+fi
+
+line_count=$(diff -y --suppress-common-lines <(jq --sort-keys . "${custom_file_folder}/valhalla_base.json") <(jq --sort-keys . "${custom_file_folder}/valhalla.json") | wc -l)
+if [[ $line_count != "0" ]]; then
+  echo "Valhalla config was not updated correctly. Check the generated config files."
+  exit 1
+fi
+
+# should not be added back 
+jq 'del(.meili.default.beta)' "${custom_file_folder}/valhalla.json" | sponge "${custom_file_folder}/valhalla.json"
+docker stop valhalla_full
+
+# new container with update_existing_config set to false
+docker run -d --name valhalla_no_config_update -p 8002:8002 -v $custom_file_folder:/custom_files -e tileset_name=$tileset_name -e use_tiles_ignore_pbf=False -e build_elevation=False -e build_admins=True -e build_time_zones=True -e build_tar=False -e server_threads=1 -e update_existing_config=False ${valhalla_image}
+wait_for_docker
+
+jq -e '.meili.default.beta' "${custom_file_folder}/valhalla.json" >/dev/null
+
+if [[ $? -eq 0 ]]; then
+  echo "valhalla.json should not have been updated but was"
+  exit 1
+fi
+
+line_count=$(diff -y --suppress-common-lines <(jq --sort-keys . "${custom_file_folder}/valhalla_base.json") <(jq --sort-keys . "${custom_file_folder}/valhalla.json") | wc -l)
+if [[ $line_count != "1" ]]; then
+  echo "valhalla.json should not have been updated but was"
+  exit 1
+fi
+
+docker stop  valhalla_no_config_update
+docker rm valhalla_no_config_update
+
 #### Add a PBF, restart and see if it worked ####
 echo "#### Add PBF test ####"
 # reset the config