build Valhalla graphs with elevation data (#26)

Dockerfile

@@ -1,5 +1,6 @@
 #--- BEGIN Usual Python stuff ---
-FROM valhalla/valhalla:run-latest as builder
+
+FROM ghcr.io/valhalla/valhalla:latest as builder
 LABEL maintainer=nils@gis-ops.com
 
 WORKDIR /app
@@ -13,7 +14,7 @@ RUN apt-get update -y > /dev/null && \
         python3-pip \
         python3-venv \
         curl > /dev/null && \
-    python -m pip install --upgrade pip
+    python -m pip install --upgrade pip --break-system-packages
 
 ENV POETRY_BIN /root/.local/bin/poetry
 
@@ -46,16 +47,16 @@ RUN cd /usr/local/bin && \
   for f in valhalla*; do rm $f; done && \
   cd .. && mv $preserve ./bin
 
-FROM ubuntu:22.04 as runner_base
+FROM ubuntu:23.04 as runner_base
 MAINTAINER Nils Nolde <nils@gis-ops.com>
 
 # install Valhalla stuff
 RUN apt-get update > /dev/null && \
     export DEBIAN_FRONTEND=noninteractive && \
     apt-get install -y libluajit-5.1-2 \
-      libzmq5 libczmq4 spatialite-bin libprotobuf-lite23 sudo locales wget \
+      libzmq5 libczmq4 spatialite-bin libprotobuf-lite32 sudo locales wget \
       libsqlite3-0 libsqlite3-mod-spatialite libcurl4 python-is-python3 osmctools \
-      python3.10-minimal python3-distutils curl unzip moreutils jq spatialite-bin supervisor > /dev/null
+      python3.11-minimal python3-distutils curl unzip moreutils jq spatialite-bin supervisor > /dev/null
 
 WORKDIR /app
 

docker-compose.yml

@@ -77,10 +77,6 @@ services:
       - worker
     healthcheck:
       disable: true
-    # set the proxies from the host env vars
-    environment:
-      http_proxy: "${http_proxy}"
-      https_proxy: "${https_proxy}"
     restart: always
   fake-smtp-server:
     image: reachfive/fake-smtp-server:latest

pyproject.toml

@@ -19,12 +19,11 @@ sqlalchemy-utils = "^0.38.2"
 passlib = "^1.7.4"
 shapely = "^1.8.1"
 gunicorn = "^20.1.0"
-gevent = "^21.12.0"
-pyproj = "^3.3.0"
 psycopg2-binary = "^2.9.3"
 fastapi = "^0.85.1"
 sqlmodel = "^0.0.8"
 arq = "^0.25.0"
+anyio = "^3.0.0"  # some anyio breaking change in 4.x breaks starlette's testclient
 uvicorn = {extras = ["standard"], version = "^0.20.0"}
 pydantic = {extras = ["email"], version = "^1.10.2"}
 osmium = "^3.5.0"

scripts/run_valhalla.sh

@@ -35,13 +35,18 @@ wait_for_lock() {
   exit 1
 }
 
+# reset config so we don't start the service with a valid graph
+# we only need the service to query the /status endpoint to decide
+# which instance to shut down/start up
 reset_config() {
   jq --arg d "" '.mjolnir.tile_dir = $d' "${valhalla_config}"| sponge "${valhalla_config}"
+  jq --arg d "" '.additional_data.elevation = $d' "${valhalla_config}"| sponge "${valhalla_config}"
 }
 
 PORT_8002="8002"
 PORT_8003="8003"
 # $DATA_DIR needs to be defined, either by supervisor or the current shell
+ELEVATION_DIR="$DATA_DIR/elevation"
 VALHALLA_DIR_8002="$DATA_DIR/osm/$PORT_8002"
 VALHALLA_DIR_8003="$DATA_DIR/osm/$PORT_8003"
 # TODO: change PBF
@@ -102,8 +107,10 @@ while true; do
   valhalla_config="$CURRENT_VALHALLA_DIR/valhalla.json"
   valhalla_build_config \
     --httpd-service-listen "tcp://*:${CURRENT_PORT}" \
+    --loki-actions "status" \
     --mjolnir-tile-extract "" \
     --mjolnir-tile-dir "$CURRENT_VALHALLA_DIR" \
+    --additional-data-elevation "$ELEVATION_DIR" \
     --mjolnir-concurrency "$CONCURRENCY" \
     --mjolnir-logging-type "" \
     > "${valhalla_config}" || exit 1
@@ -124,8 +131,16 @@ while true; do
     continue
   fi
 
-  echo "INFO: Running build tiles with: $PBF"
-  valhalla_build_tiles -c "${valhalla_config}" "$PBF" || exit 1
+  echo "INFO: Building initial graph with $PBF..."
+  valhalla_build_tiles -c "${valhalla_config}" -s initialize -e build "$PBF" || exit 1
+
+  echo "INFO: Downloading elevation to $ELEVATION_DIR..."
+  valhalla_build_elevation --from-tiles --decompress -c ${valhalla_config} -v || exit 1
+
+  echo "INFO: Enhancing initial tiles with elevation..."
+  valhalla_build_tiles -c "${valhalla_config}" -s enhance -e cleanup "$PBF" || exit 1
+
+  # reset config so the service won't load the graph
   reset_config
 
   echo "INFO: Updating the registered packages with $(which python3)"

tests/conftest.py

@@ -12,7 +12,7 @@
 from routing_packager_app.worker import create_package
 
 
-@pytest.yield_fixture(scope="session", autouse=True)
+@pytest.fixture(scope="session", autouse=True)
 def create_worker():
     """Auto-runs and creates an event loop for the worker."""
     worker = Worker([create_package])
@@ -35,7 +35,7 @@ def get_client(get_app: FastAPI) -> TestClient:
     return client
 
 
-@pytest.yield_fixture(scope="session", autouse=True)
+@pytest.fixture(scope="session", autouse=True)
 def create_db():
     from routing_packager_app.db import engine
     from routing_packager_app.api_v1.models import User
@@ -46,7 +46,7 @@ def create_db():
     SQLModel.metadata.drop_all(engine, checkfirst=True)
 
 
-@pytest.yield_fixture(scope="function")
+@pytest.fixture(scope="function")
 def get_session():
     from routing_packager_app.db import engine
 
@@ -69,7 +69,7 @@ def basic_auth_header():
 
 
 # Creates needed directories and removes them after the test function
-@pytest.yield_fixture(scope="session", autouse=True)
+@pytest.fixture(scope="session", autouse=True)
 def handle_dirs():
     paths = [SETTINGS.get_valhalla_path(p) for p in (8002, 8003)]
     for p in paths:

tests/jobs/conftest.py

@@ -14,7 +14,7 @@
 HTTPServer.DEFAULT_LISTEN_HOST = "localhost"
 
 
-@pytest.yield_fixture(scope="function", autouse=True)
+@pytest.fixture(scope="function", autouse=True)
 def delete_jobs(get_session: Session):
     yield
     jobs = get_session.exec(select(Job)).all()
@@ -23,14 +23,14 @@ def delete_jobs(get_session: Session):
         get_session.commit()
 
 
-@pytest.yield_fixture(scope="function", autouse=True)
+@pytest.fixture(scope="function", autouse=True)
 def delete_dirs():
     yield
     for dir_ in SETTINGS.get_output_path().iterdir():
         rmtree(dir_)
 
 
-@pytest.yield_fixture(scope="function")
+@pytest.fixture(scope="function")
 def copy_valhalla_tiles():
     for dir_ in SETTINGS.get_output_path().parent.joinpath("andorra_tiles").iterdir():
         copytree(dir_, SETTINGS.get_valhalla_path(8002).joinpath(dir_.stem))