From 28b6c815743d5ef09917c4d7dc110192aef21ed9 Mon Sep 17 00:00:00 2001 From: Ellen Poe Date: Thu, 11 Aug 2022 18:39:46 -0700 Subject: [PATCH 1/5] Switch build system to earthly --- Earthfile | 483 ++++++++++++++++++ Makefile | 213 -------- docker-compose.yaml | 105 +++- geocoder/nominatim/Dockerfile | 9 - geocoder/nominatim/restore_then_start.sh | 24 - geocoder/nominatim_build/.gitignore | 1 - geocoder/nominatim_build/Dockerfile | 19 - geocoder/photon/.gitignore | 1 - geocoder/photon/Dockerfile | 25 - geocoder/photon/untar_and_start.sh | 11 - geocoder/photon_build/.gitignore | 1 - geocoder/photon_build/Dockerfile | 35 -- gtfs/download/.gitignore | 1 - gtfs/download/Dockerfile | 14 - gtfs/enumerate/Dockerfile | 15 - mbtiles_build/.gitignore | 1 - mbtiles_build/Dockerfile | 30 -- mbtiles_build/download_mirrored_data.sh | 8 - otp/build/.gitignore | 1 - otp/build/Dockerfile | 31 -- otp/build/build-config.json | 3 - otp/run/Dockerfile | 21 - .../gtfs}/download_gtfs_feeds.py | 0 .../gtfs}/enumerate_gtfs_feeds.py | 0 services/init_container_dl.sh | 15 + services/nginx/init.sh | 17 + .../nominatim/dump.sh | 4 +- services/nominatim/init.sh | 52 ++ services/nominatim/run.sh | 7 + services/otp/init.sh | 17 + {otp/build => services/otp}/maybe_build.sh | 0 .../otp/run_otp.sh | 3 +- .../photon/import_and_dump.sh | 7 +- services/photon/init.sh | 25 + .../tileserver}/assets/.dockerignore | 0 .../tileserver}/assets/.gitignore | 0 .../tileserver}/assets/COPYRIGHT_NOTICE.md | 0 .../tileserver}/assets/Dockerfile | 0 .../tileserver}/assets/NotoSans-Bold.ttf | Bin .../tileserver}/assets/NotoSans-Italic.ttf | Bin .../tileserver}/assets/NotoSans-Regular.ttf | Bin .../tileserver}/assets/OFL.txt | 0 .../tileserver}/assets/build_glyphs.js | 0 .../tileserver}/assets/build_sprites.js | 0 .../tileserver}/assets/package-lock.json | 0 .../tileserver}/assets/package.json | 0 .../tileserver}/assets/sprites/LICENSE.md | 0 .../assets/sprites/airfield_11.svg | 0 .../tileserver}/assets/sprites/airport_11.svg | 0 .../assets/sprites/alcohol_shop_11.svg | 0 .../assets/sprites/amusement_park_11.svg | 0 .../assets/sprites/aquarium_11.svg | 0 .../assets/sprites/art_gallery_11.svg | 0 .../assets/sprites/attraction_11.svg | 0 .../tileserver}/assets/sprites/bakery_11.svg | 0 .../tileserver}/assets/sprites/bank_11.svg | 0 .../tileserver}/assets/sprites/bar_11.svg | 0 .../tileserver}/assets/sprites/beer_11.svg | 0 .../tileserver}/assets/sprites/bicycle_11.svg | 0 .../assets/sprites/bicycle_rental_11.svg | 0 .../tileserver}/assets/sprites/bus_11.svg | 0 .../tileserver}/assets/sprites/cafe_11.svg | 0 .../assets/sprites/campsite_11.svg | 0 .../tileserver}/assets/sprites/car_11.svg | 0 .../tileserver}/assets/sprites/castle_11.svg | 0 .../assets/sprites/cemetery_11.svg | 0 .../tileserver}/assets/sprites/cinema_11.svg | 0 .../tileserver}/assets/sprites/circle_11.svg | 0 .../assets/sprites/circle_stroked_11.svg | 0 .../assets/sprites/clothing_store_11.svg | 0 .../tileserver}/assets/sprites/college_11.svg | 0 .../tileserver}/assets/sprites/dentist_11.svg | 0 .../tileserver}/assets/sprites/doctor_11.svg | 0 .../assets/sprites/dog_park_11.svg | 0 .../assets/sprites/drinking_water_11.svg | 0 .../tileserver}/assets/sprites/embassy_11.svg | 0 .../assets/sprites/entrance_11.svg | 0 .../assets/sprites/fast_food_11.svg | 0 .../assets/sprites/ferry_terminal_11.svg | 0 .../assets/sprites/fire_station_11.svg | 0 .../tileserver}/assets/sprites/fuel_11.svg | 0 .../tileserver}/assets/sprites/garden_11.svg | 0 .../tileserver}/assets/sprites/golf_11.svg | 0 .../tileserver}/assets/sprites/grocery_11.svg | 0 .../tileserver}/assets/sprites/harbor_11.svg | 0 .../assets/sprites/heliport_11.svg | 0 .../assets/sprites/hospital_11.svg | 0 .../assets/sprites/ice_cream_11.svg | 0 .../assets/sprites/information_11.svg | 0 .../tileserver}/assets/sprites/laundry_11.svg | 0 .../tileserver}/assets/sprites/library_11.svg | 0 .../tileserver}/assets/sprites/lodging_11.svg | 0 .../tileserver}/assets/sprites/marker_11.svg | 0 .../assets/sprites/monument_11.svg | 0 .../assets/sprites/mountain_11.svg | 0 .../tileserver}/assets/sprites/museum_11.svg | 0 .../tileserver}/assets/sprites/music_11.svg | 0 .../tileserver}/assets/sprites/oneway.svg | 0 .../tileserver}/assets/sprites/park_11.svg | 0 .../assets/sprites/pharmacy_11.svg | 0 .../assets/sprites/picnic_site_11.svg | 0 .../tileserver}/assets/sprites/pitch_11.svg | 0 .../assets/sprites/place_of_worship_11.svg | 0 .../assets/sprites/playground_11.svg | 0 .../tileserver}/assets/sprites/police_11.svg | 0 .../tileserver}/assets/sprites/post_11.svg | 0 .../tileserver}/assets/sprites/prison_11.svg | 0 .../assets/sprites/rail_light_11.svg | 0 .../assets/sprites/rail_metro_11.svg | 0 .../tileserver}/assets/sprites/railway_11.svg | 0 .../assets/sprites/religious_christian_11.svg | 0 .../assets/sprites/religious_jewish_11.svg | 0 .../assets/sprites/religious_muslim_11.svg | 0 .../assets/sprites/restaurant_11.svg | 0 .../tileserver}/assets/sprites/road_1.svg | 0 .../tileserver}/assets/sprites/road_2.svg | 0 .../tileserver}/assets/sprites/road_3.svg | 0 .../tileserver}/assets/sprites/road_4.svg | 0 .../tileserver}/assets/sprites/road_5.svg | 0 .../tileserver}/assets/sprites/road_6.svg | 0 .../tileserver}/assets/sprites/rocket_11.svg | 0 .../tileserver}/assets/sprites/school_11.svg | 0 .../tileserver}/assets/sprites/shop_11.svg | 0 .../tileserver}/assets/sprites/stadium_11.svg | 0 .../tileserver}/assets/sprites/star_11.svg | 0 .../assets/sprites/suitcase_11.svg | 0 .../assets/sprites/swimming_11.svg | 0 .../tileserver}/assets/sprites/theatre_11.svg | 0 .../tileserver}/assets/sprites/toilet_11.svg | 0 .../assets/sprites/town_hall_11.svg | 0 .../assets/sprites/triangle_11.svg | 0 .../assets/sprites/triangle_stroked_11.svg | 0 .../assets/sprites/us-highway_1.svg | 0 .../assets/sprites/us-highway_2.svg | 0 .../assets/sprites/us-highway_3.svg | 0 .../assets/sprites/us-interstate_1.svg | 0 .../assets/sprites/us-interstate_2.svg | 0 .../assets/sprites/us-interstate_3.svg | 0 .../tileserver}/assets/sprites/us-state_1.svg | 0 .../tileserver}/assets/sprites/us-state_2.svg | 0 .../tileserver}/assets/sprites/us-state_3.svg | 0 .../tileserver}/assets/sprites/us-state_4.svg | 0 .../tileserver}/assets/sprites/us-state_5.svg | 0 .../tileserver}/assets/sprites/us-state_6.svg | 0 .../assets/sprites/veterinary_11.svg | 0 .../tileserver}/assets/sprites/volcano_11.svg | 0 .../tileserver}/assets/sprites/wave.svg | 0 .../tileserver}/assets/sprites/zoo_11.svg | 0 .../tileserver}/config.json.template | 4 +- .../tileserver}/configure_run.sh | 2 +- services/tileserver/init.sh | 43 ++ .../tileserver}/style/LICENSE.md | 0 .../tileserver}/style/style.json.template | 0 services/valhalla/init.sh | 27 + tileserver/image/Dockerfile | 18 - valhalla/build/.gitignore | 1 - valhalla/build/Dockerfile | 16 - valhalla/run/Dockerfile | 15 - valhalla/run/gen_hash.sh | 11 - .../{.dockerignore => .earthlyignore} | 0 web/frontend/src/components/BaseMap.vue | 2 +- 161 files changed, 790 insertions(+), 548 deletions(-) create mode 100644 Earthfile delete mode 100644 Makefile delete mode 100644 geocoder/nominatim/Dockerfile delete mode 100644 geocoder/nominatim/restore_then_start.sh delete mode 100644 geocoder/nominatim_build/.gitignore delete mode 100644 geocoder/nominatim_build/Dockerfile delete mode 100644 geocoder/photon/.gitignore delete mode 100644 geocoder/photon/Dockerfile delete mode 100755 geocoder/photon/untar_and_start.sh delete mode 100644 geocoder/photon_build/.gitignore delete mode 100644 geocoder/photon_build/Dockerfile delete mode 100644 gtfs/download/.gitignore delete mode 100644 gtfs/download/Dockerfile delete mode 100644 gtfs/enumerate/Dockerfile delete mode 100644 mbtiles_build/.gitignore delete mode 100644 mbtiles_build/Dockerfile delete mode 100755 mbtiles_build/download_mirrored_data.sh delete mode 100644 otp/build/.gitignore delete mode 100644 otp/build/Dockerfile delete mode 100644 otp/build/build-config.json delete mode 100644 otp/run/Dockerfile rename {gtfs/download => services/gtfs}/download_gtfs_feeds.py (100%) rename {gtfs/enumerate => services/gtfs}/enumerate_gtfs_feeds.py (100%) create mode 100644 services/init_container_dl.sh create mode 100755 services/nginx/init.sh rename geocoder/nominatim_build/import_then_dump.sh => services/nominatim/dump.sh (66%) create mode 100755 services/nominatim/init.sh create mode 100755 services/nominatim/run.sh create mode 100755 services/otp/init.sh rename {otp/build => services/otp}/maybe_build.sh (100%) rename otp/run/rename_and_run.sh => services/otp/run_otp.sh (67%) rename geocoder/photon_build/import_from_dump.sh => services/photon/import_and_dump.sh (72%) create mode 100755 services/photon/init.sh rename {tileserver => services/tileserver}/assets/.dockerignore (100%) rename {tileserver => services/tileserver}/assets/.gitignore (100%) rename {tileserver => services/tileserver}/assets/COPYRIGHT_NOTICE.md (100%) rename {tileserver => services/tileserver}/assets/Dockerfile (100%) rename {tileserver => services/tileserver}/assets/NotoSans-Bold.ttf (100%) rename {tileserver => services/tileserver}/assets/NotoSans-Italic.ttf (100%) rename {tileserver => services/tileserver}/assets/NotoSans-Regular.ttf (100%) rename {tileserver => services/tileserver}/assets/OFL.txt (100%) rename {tileserver => services/tileserver}/assets/build_glyphs.js (100%) rename {tileserver => services/tileserver}/assets/build_sprites.js (100%) rename {tileserver => services/tileserver}/assets/package-lock.json (100%) rename {tileserver => services/tileserver}/assets/package.json (100%) rename {tileserver => services/tileserver}/assets/sprites/LICENSE.md (100%) rename {tileserver => services/tileserver}/assets/sprites/airfield_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/airport_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/alcohol_shop_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/amusement_park_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/aquarium_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/art_gallery_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/attraction_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/bakery_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/bank_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/bar_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/beer_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/bicycle_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/bicycle_rental_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/bus_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/cafe_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/campsite_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/car_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/castle_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/cemetery_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/cinema_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/circle_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/circle_stroked_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/clothing_store_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/college_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/dentist_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/doctor_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/dog_park_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/drinking_water_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/embassy_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/entrance_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/fast_food_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/ferry_terminal_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/fire_station_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/fuel_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/garden_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/golf_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/grocery_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/harbor_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/heliport_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/hospital_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/ice_cream_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/information_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/laundry_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/library_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/lodging_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/marker_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/monument_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/mountain_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/museum_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/music_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/oneway.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/park_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/pharmacy_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/picnic_site_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/pitch_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/place_of_worship_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/playground_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/police_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/post_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/prison_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/rail_light_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/rail_metro_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/railway_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/religious_christian_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/religious_jewish_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/religious_muslim_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/restaurant_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/road_1.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/road_2.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/road_3.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/road_4.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/road_5.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/road_6.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/rocket_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/school_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/shop_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/stadium_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/star_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/suitcase_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/swimming_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/theatre_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/toilet_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/town_hall_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/triangle_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/triangle_stroked_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-highway_1.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-highway_2.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-highway_3.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-interstate_1.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-interstate_2.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-interstate_3.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-state_1.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-state_2.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-state_3.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-state_4.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-state_5.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/us-state_6.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/veterinary_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/volcano_11.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/wave.svg (100%) rename {tileserver => services/tileserver}/assets/sprites/zoo_11.svg (100%) rename {tileserver/image => services/tileserver}/config.json.template (86%) rename {tileserver/image => services/tileserver}/configure_run.sh (67%) create mode 100755 services/tileserver/init.sh rename {tileserver => services/tileserver}/style/LICENSE.md (100%) rename {tileserver => services/tileserver}/style/style.json.template (100%) create mode 100755 services/valhalla/init.sh delete mode 100644 tileserver/image/Dockerfile delete mode 100644 valhalla/build/.gitignore delete mode 100644 valhalla/build/Dockerfile delete mode 100644 valhalla/run/Dockerfile delete mode 100755 valhalla/run/gen_hash.sh rename web/frontend/{.dockerignore => .earthlyignore} (100%) diff --git a/Earthfile b/Earthfile new file mode 100644 index 000000000..c373613a4 --- /dev/null +++ b/Earthfile @@ -0,0 +1,483 @@ +VERSION 0.6 + + +############################## +# OSM extract +############################## + +metro: + FROM debian:bullseye-slim + ARG area + COPY (+extract/data.osm.pbf --area=${area}) /data.osm.pbf + COPY (+nominatim-build/tokenizer.tar --area=${area}) /tokenizer.tar + COPY (+nominatim-build/nominatim.sql.bz2 --area=${area}) /nominatim.tar.bz2 + COPY (+photon-build/photon.tar.bz2 --area=${area}) /photon.tar.bz2 + COPY (+gtfs-build/gtfs.tar --area=${area}) /gtfs.tar + COPY (+otp-build/graph.obj --area=${area}) /graph.obj + COPY (+valhalla-build/valhalla.tar.bz2 --area=${area}) /valhalla.tar.bz2 + COPY (+planetiler-build-mbtiles/output.mbtiles --area=${area}) /output.mbtiles + SAVE ARTIFACT /data.osm.pbf AS LOCAL ./data/${area}.osm.pbf + SAVE ARTIFACT /tokenizer.tar AS LOCAL ./data/${area}.tokenizer.tar + SAVE ARTIFACT /nominatim.tar.bz2 AS LOCAL ./data/${area}.nominatim.sql.bz2 + SAVE ARTIFACT /photon.tar.bz2 AS LOCAL ./data/${area}.photon.tar.bz2 + SAVE ARTIFACT /gtfs.tar AS LOCAL ./data/${area}.gtfs.tar + SAVE ARTIFACT /graph.obj AS LOCAL ./data/${area}.graph.obj + SAVE ARTIFACT /valhalla.tar.bz2 AS LOCAL ./data/${area}.valhalla.tar.bz2 + SAVE ARTIFACT /output.mbtiles AS LOCAL ./data/${area}.mbtiles + BUILD +images + +images: + FROM debian:bullseye-slim + COPY (+tileserver-build/fonts.tar) /fonts.tar + COPY (+tileserver-build/sprite.tar) /sprite.tar + SAVE ARTIFACT /fonts.tar AS LOCAL ./data/fonts.tar + SAVE ARTIFACT /sprite.tar AS LOCAL ./data/sprite.tar + BUILD +nominatim-serve-image + BUILD +photon-serve-image + BUILD +otp-serve-image + BUILD +valhalla-serve-image + BUILD +web-serve-image + BUILD +tileserver-serve-image + BUILD +nominatim-init-image + BUILD +photon-init-image + BUILD +otp-init-image + BUILD +valhalla-init-image + BUILD +web-init-image + BUILD +tileserver-init-image + +extract: + FROM +downloader-base + ARG area + COPY --if-exists ${area}.osm.pbf /data/data.osm.pbf + IF [ ! -f "/data/data.osm.pbf" ] + RUN wget -U headway/1.0 -O /data/data.osm.pbf "https://download.bbbike.org/osm/bbbike/${area}/${area}.osm.pbf" + END + SAVE ARTIFACT /data/data.osm.pbf /data.osm.pbf + +############################## +# Nominatim +############################## + +nominatim-base-image: + FROM mediagis/nominatim:4.0 + ENV TZ="America/New_York" + RUN apt-get update -y && apt-get install -y pbzip2 python3-pip sudo + +nominatim-build: + FROM +nominatim-base-image + ARG area + COPY (+extract/data.osm.pbf --area=${area}) /data.osm.pbf + + RUN bash -c 'useradd -m -p ${NOMINATIM_PASSWORD} nominatim' + ENV PBF_PATH=/data.osm.pbf + RUN bash -c '/app/config.sh' + RUN bash -c '/app/init.sh' + + RUN touch /var/lib/postgresql/12/main/import-finished + + COPY ./services/nominatim/dump.sh /app + + RUN /app/dump.sh + SAVE ARTIFACT /nominatim/tokenizer.tar /tokenizer.tar + SAVE ARTIFACT /dump/nominatim.sql.bz2 /nominatim.sql.bz2 + +nominatim-init-image: + FROM +nominatim-base-image + RUN apt-get update -y && apt-get install -y wget ca-certificates + + WORKDIR /services + COPY ./services/nominatim/init.sh /services/ + + CMD ["/services/init.sh"] + + SAVE IMAGE headway_nominatim_init + +nominatim-serve-image: + FROM +nominatim-base-image + COPY ./services/nominatim/run.sh /app/run-custom.sh + + ENTRYPOINT ["/bin/bash"] + CMD ["/app/run-custom.sh"] + SAVE IMAGE headway_nominatim + +############################## +# Photon +############################## + +photon-download: + FROM +downloader-base + ARG PHOTON_VERSION=0.3.5 + ARG PHOTON_HASH=6f438e905b020a70ec54de47b7c7f8b9c7921ef0ee84a3a9e0a6a6ebb7f74fa8 + + RUN wget -O /data/photon.jar "https://github.com/komoot/photon/releases/download/${PHOTON_VERSION}/photon-${PHOTON_VERSION}.jar" + RUN echo "${PHOTON_HASH} /data/photon.jar" | sha256sum --check + + SAVE ARTIFACT /data/photon.jar /photon.jar + +photon-base-image: + FROM +java-base + + RUN mkdir /photon + RUN useradd -ms /bin/bash photon + RUN chown photon /photon + + COPY +photon-download/photon.jar /photon/photon.jar + +photon-build: + FROM +nominatim-base-image + RUN apt-get update \ + && apt-get install -y --no-install-recommends openjdk-17-jre-headless + + RUN mkdir /photon + RUN useradd -ms /bin/bash photon + COPY +photon-download/photon.jar /photon/photon.jar + RUN chown -R photon /photon + + RUN chown -R photon /app + + ARG area + ARG langs="es,fr,de,en,it" + ENV HEADWAY_PHOTON_LANGUAGES=${langs} + COPY ./services/photon/import_and_dump.sh /app/ + COPY (+nominatim-build/nominatim.sql.bz2 --area=${area}) /nominatim_data/data.nominatim.sql.bz2 + + USER root + RUN /app/import_and_dump.sh + + SAVE ARTIFACT /photon/photon.tar.bz2 /photon.tar.bz2 + +photon-init-image: + FROM +photon-base-image + RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql postgis postgresql-postgis wget pbzip2 + + WORKDIR /services + COPY ./services/photon/init.sh /services/ + + USER root + CMD ["/services/init.sh"] + + SAVE IMAGE headway_photon_init + +photon-serve-image: + FROM +photon-base-image + WORKDIR /photon + USER photon + CMD ["java", "-jar", "/photon/photon.jar"] + + SAVE IMAGE headway_photon + +############################## +# Planetiler +############################## + +planetiler-download-mirrored-data: + FROM +downloader-base + WORKDIR /data + RUN wget https://f000.backblazeb2.com/file/headway/sources.tar && tar xvf sources.tar && rm sources.tar + SAVE ARTIFACT /data/lake_centerline.shp.zip /lake_centerline.shp.zip + SAVE ARTIFACT /data/natural_earth_vector.sqlite.zip /natural_earth_vector.sqlite.zip + SAVE ARTIFACT /data/water-polygons-split-3857.zip /water-polygons-split-3857.zip + +planetiler-download: + FROM +downloader-base + ARG PLANETILER_VERSION=v0.5.0 + ARG PLANETILER_HASH=5f08d8f351751373084b1c2abd21bb38cbf66357dd2a02d2692d3561f16db70b + + RUN wget -O /data/planetiler.jar https://github.com/onthegomap/planetiler/releases/download/${PLANETILER_VERSION}/planetiler.jar + RUN ls -l /data + RUN echo "${PLANETILER_HASH} /data/planetiler.jar" | sha256sum --check + + SAVE ARTIFACT /data/planetiler.jar /planetiler.jar + +planetiler-image: + FROM +java-base + COPY +planetiler-download/planetiler.jar /planetiler/planetiler.jar + COPY +planetiler-download-mirrored-data/lake_centerline.shp.zip /data/sources/ + COPY +planetiler-download-mirrored-data/natural_earth_vector.sqlite.zip /data/sources/ + COPY +planetiler-download-mirrored-data/water-polygons-split-3857.zip /data/sources/ + +planetiler-build-mbtiles: + FROM +planetiler-image + WORKDIR / + ARG area + COPY (+extract/data.osm.pbf --area=${area}) /data/ + RUN sha256sum /planetiler/planetiler.jar && java -jar /planetiler/planetiler.jar --force osm_path=/data/data.osm.pbf + SAVE ARTIFACT /data/output.mbtiles /output.mbtiles + +############################## +# GTFS +############################## + +gtfs-base: + FROM python:3 + RUN pip install requests + WORKDIR /gtfs + RUN mkdir /gtfs_feeds + +gtfs-enumerate: + FROM +gtfs-base + COPY ./services/gtfs/enumerate_gtfs_feeds.py /gtfs/ + ARG area + ARG bbox + ENV BBOX=${bbox} + IF [ ! -z "${BBOX}" ] + ENV HEADWAY_BBOX=${BBOX} + RUN python /gtfs/enumerate_gtfs_feeds.py + ELSE + COPY web/bboxes.csv /gtfs/bboxes.csv + RUN bash -c "export HEADWAY_BBOX=\"$(grep "${area}:" /gtfs/bboxes.csv | cut -d':' -f2)\" && python /gtfs/enumerate_gtfs_feeds.py" + END + SAVE ARTIFACT /gtfs_feeds/gtfs_feeds.csv /gtfs_feeds.csv AS LOCAL ./data/${area}.gtfs_feeds.csv + +gtfs-build: + FROM +gtfs-base + COPY ./services/gtfs/download_gtfs_feeds.py /gtfs/ + ARG area + COPY --if-exists build/${area}.gtfs_feeds.csv /gtfs/gtfs_feeds.csv + RUN touch /gtfs/gtfs_feeds.csv # Just in case the GTFS feeds weren't enumerated earlier. + RUN python /gtfs/download_gtfs_feeds.py + RUN bash -c "cd /gtfs_feeds && ls *.zip | tar -cf /gtfs/gtfs.tar --files-from -" + SAVE ARTIFACT /gtfs/gtfs.tar /gtfs.tar + +############################## +# OpenTripPlanner +############################## + +otp-download: + FROM +downloader-base + + ARG OTP_VERISON=2.1.0 + ARG OTP_HASH=b4c986b1c726c7d81d255fa183d32576122ba4e50290d53e4bb40be051971134 + + RUN wget -O /data/otp-shaded.jar "https://github.com/opentripplanner/OpenTripPlanner/releases/download/v${OTP_VERISON}/otp-${OTP_VERISON}-shaded.jar" + RUN echo "${OTP_HASH} /data/otp-shaded.jar" | sha256sum --check + + SAVE ARTIFACT /data/otp-shaded.jar /otp-shaded.jar + +otp-base: + FROM +java11-base + + RUN mkdir /data + RUN mkdir /otp + + COPY +otp-download/otp-shaded.jar /otp + +otp-build: + FROM +otp-base + + ARG area + + COPY (+gtfs-build/gtfs.tar --area=${area}) /data/ + COPY (+extract/data.osm.pbf --area=${area}) /data/ + WORKDIR /data + RUN tar xvf gtfs.tar + + ARG javaMemArgs=-Xmx4G + COPY ./services/otp/maybe_build.sh /otp + + RUN /otp/maybe_build.sh + + SAVE ARTIFACT /data/graph.obj /graph.obj + +otp-init-image: + FROM debian:bullseye-slim + COPY ./services/otp/init.sh /app/init.sh + CMD ["/app/init.sh"] + SAVE IMAGE headway_otp_init + +otp-serve-image: + FROM +otp-base + + ARG javaMemArgs=-Xmx4G + COPY ./services/otp/run_otp.sh /otp + + CMD ["/otp/run_otp.sh"] + SAVE IMAGE headway_otp + +############################## +# Valhalla +############################## + +valhalla-base-image: + FROM gisops/valhalla:latest + + USER root + RUN apt-get -y update && apt-get install -y pbzip2 + WORKDIR /tiles + RUN chown valhalla /tiles + USER valhalla + +valhalla-build: + FROM +valhalla-base-image + + RUN valhalla_build_config --mjolnir-tile-dir /tiles --mjolnir-timezone /tiles/timezones.sqlite --mjolnir-admin /tiles/admins.sqlite > valhalla.json + RUN valhalla_build_timezones > /tiles/timezones.sqlite + + ARG area + COPY (+extract/data.osm.pbf --area=${area}) /tiles/data.osm.pbf + + RUN valhalla_build_tiles -c valhalla.json /tiles/data.osm.pbf + + RUN bash -c 'cd /tiles && ls | tar -c --files-from - | pbzip2 -c -6 > /tiles/valhalla.tar.bz2' + SAVE ARTIFACT /tiles/valhalla.tar.bz2 /valhalla.tar.bz2 + +valhalla-init-image: + FROM +valhalla-base-image + COPY ./services/valhalla/init.sh /app/init.sh + ENTRYPOINT ["/bin/bash"] + USER root + CMD ["/app/init.sh"] + SAVE IMAGE headway_valhalla_init + +valhalla-serve-image: + FROM +valhalla-base-image + ENTRYPOINT ["valhalla_service"] + USER valhalla + CMD ["/data/valhalla.json"] + SAVE IMAGE headway_valhalla + +############################## +# tileserver-gl-light +############################## + +tileserver-build: + FROM node:12-slim + RUN npm install fontnik + + COPY ./services/tileserver/assets/build_glyphs.js \ + ./services/tileserver/assets/build_sprites.js \ + ./services/tileserver/assets/package.json \ + ./services/tileserver/assets/package-lock.json \ + ./services/tileserver/assets/*.ttf \ + /app/ + + WORKDIR /app + RUN npm install + RUN mkdir -p /app/sprite/ + COPY ./services/tileserver/assets/sprites/*.svg /app/sprite/ + WORKDIR /app + RUN mkdir /output + RUN useradd -ms /bin/bash fontnik + RUN chown fontnik /output + + USER fontnik + + RUN mkdir "/output/Noto Sans Bold" + RUN mkdir "/output/Noto Sans Italic" + RUN mkdir "/output/Noto Sans Regular" + + RUN node build_glyphs NotoSans-Bold.ttf "/output/Noto Sans Bold" + RUN node build_glyphs NotoSans-Italic.ttf "/output/Noto Sans Italic" + RUN node build_glyphs NotoSans-Regular.ttf "/output/Noto Sans Regular" + + RUN node build_sprites /output/sprite /app/sprite + RUN node build_sprites --retina /output/sprite@2x /app/sprite + + WORKDIR /output + + RUN tar -cf fonts.tar "Noto Sans Bold" "Noto Sans Italic" "Noto Sans Regular" + RUN tar -cf sprite.tar sprite.json sprite.png sprite@2x.json sprite@2x.png + + SAVE ARTIFACT /output/fonts.tar /fonts.tar + SAVE ARTIFACT /output/sprite.tar /sprite.tar + +tileserver-init-image: + FROM debian:bullseye-slim + RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates wget pbzip2 + + COPY ./services/tileserver/init.sh /app/init.sh + CMD ["/app/init.sh"] + SAVE IMAGE headway_tileserver_init + +tileserver-serve-image: + FROM node:16 + + RUN npm install -g tileserver-gl-light + + USER root + + RUN apt-get update -y && apt-get install -y gettext-base + + RUN mkdir -p /app/styles + RUN mkdir -p /styles + RUN chown -R node /app + RUN chown -R node /styles + USER node + + COPY ./services/tileserver/style/style.json.template /styles/style.json.template + + COPY ./services/tileserver/configure_run.sh ./services/tileserver/config.json.template /app/ + + CMD ["/app/configure_run.sh"] + SAVE IMAGE headway_tileserver + +############################## +# Web +############################## + +web-build: + FROM node:16-slim + RUN yarn global add @quasar/cli + COPY ./web/frontend /frontend + WORKDIR /frontend + RUN yarn install && quasar build + SAVE ARTIFACT /frontend/dist/spa /spa + +web-init-image: + FROM debian:bullseye-slim + COPY ./services/nginx/init.sh /app/init.sh + CMD ["/app/init.sh"] + SAVE IMAGE headway_nginx_init + +web-serve-image: + FROM nginx + + COPY web/init.sh web/bboxes.csv /frontend/ + + COPY +web-build/spa /usr/share/nginx/html/ + + COPY web/nginx.conf.template /etc/nginx/templates/nginx.conf.template + + ENV HEADWAY_PUBLIC_URL=http://127.0.0.1:8080 + ENV HEADWAY_BBOX_PATH=/frontend/bbox.txt + ENV HEADWAY_CAPABILITIES_PATH=/frontend/capabilities.txt + ENV HEADWAY_SHARED_VOL=/data + ENV HEADWAY_HTTP_PORT=8080 + ENV HEADWAY_RESOLVER=127.0.0.11 + ENV HEADWAY_OTP_URL=http://otp:8080 + ENV HEADWAY_VALHALLA_URL=http://valhalla:8002 + ENV HEADWAY_PHOTON_URL=http://photon:2322 + ENV HEADWAY_TILESERVER_URL=http://mbtileserver:8000 + ENV HEADWAY_NOMINATIM_URL=http://nominatim:8080 + # for escaping $ in nginx template + ENV ESC=$ + ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx + ENTRYPOINT ["/frontend/init.sh"] + + SAVE IMAGE headway_nginx + + +############################## +# Generic base images +############################## + +downloader-base: + FROM debian:bullseye-slim + ENV TZ="America/New_York" + RUN apt-get update \ + && apt-get install -y --no-install-recommends wget ca-certificates + RUN mkdir /data + +java-base: + FROM debian:bullseye-slim + ENV TZ="America/New_York" + RUN apt-get update \ + && apt-get install -y --no-install-recommends openjdk-17-jre-headless sudo + +java11-base: + FROM debian:bullseye-slim + ENV TZ="America/New_York" + RUN apt-get update \ + && apt-get install -y --no-install-recommends openjdk-11-jre-headless sudo diff --git a/Makefile b/Makefile deleted file mode 100644 index 50050d3df..000000000 --- a/Makefile +++ /dev/null @@ -1,213 +0,0 @@ -CITIES = Aachen Aarhus Adelaide Albuquerque Alexandria Amsterdam Antwerpen Arnhem Auckland Augsburg Austin Baghdad \ - Baku Balaton Bamberg Bangkok Barcelona Basel Beijing Beirut Berkeley Berlin Bern Bielefeld Birmingham Bochum \ - Bogota Bombay Bonn Bordeaux Boulder BrandenburgHavel Braunschweig Bremen Bremerhaven Brisbane Bristol Brno \ - Bruegge Bruessel Budapest BuenosAires Cairo Calgary Cambridge CambridgeMa Canberra CapeTown Chemnitz Chicago \ - ClermontFerrand Colmar Copenhagen Cork Corsica Corvallis Cottbus Cracow CraterLake Curitiba Cusco Dallas \ - Darmstadt Davis DenHaag Denver Dessau Dortmund Dresden Dublin Duesseldorf Duisburg Edinburgh Eindhoven Emden \ - Erfurt Erlangen Eugene Flensburg FortCollins Frankfurt FrankfurtOder Freiburg Gdansk Genf Gent Gera Glasgow \ - Gliwice Goerlitz Goeteborg Goettingen Graz Groningen Halifax Halle Hamburg Hamm Hannover Heilbronn Helsinki \ - Hertogenbosch Huntsville Innsbruck Istanbul Jena Jerusalem Johannesburg Kaiserslautern Karlsruhe Kassel \ - Katowice Kaunas Kiel Kiew Koblenz Koeln Konstanz LakeGarda LaPaz LaPlata Lausanne Leeds Leipzig Lima Linz \ - Lisbon Liverpool Ljubljana Lodz London Luebeck Luxemburg Lyon Maastricht Madison Madrid Magdeburg Mainz \ - Malmoe Manchester Mannheim Marseille Melbourne Memphis MexicoCity Miami Minsk Moenchengladbach Montevideo \ - Montpellier Montreal Moscow Muenchen Muenster NewDelhi NewOrleans NewYork Nuernberg Oldenburg Oranienburg \ - Orlando Oslo Osnabrueck Ostrava Ottawa Paderborn Palma PaloAlto Paris Perth Philadelphia PhnomPenh Portland \ - PortlandME Porto PortoAlegre Potsdam Poznan Prag Providence Regensburg Riga RiodeJaneiro Rostock Rotterdam \ - Ruegen Saarbruecken Sacramento Saigon Salzburg SanFrancisco SanJose SanktPetersburg SantaBarbara SantaCruz \ - Santiago Sarajewo Schwerin Seattle Seoul Sheffield Singapore Sofia Stockholm Stockton Strassburg Stuttgart \ - Sucre Sydney Szczecin Tallinn Tehran Tilburg Tokyo Toronto Toulouse Trondheim Tucson Turin UlanBator Ulm \ - Usedom Utrecht Vancouver Victoria WarenMueritz Warsaw WashingtonDC Waterloo Wien Wroclaw Wuerzburg Wuppertal \ - Zagreb Zuerich - -.DEFAULT_GOAL := help -.SECONDARY: -SHELL := /bin/bash -.SHELLFLAGS := -ec -DATA_DIR := ${PWD}/data -DOCKER_MEMORY := "12G" -JAVA_TOOL_OPTIONS := "-Xmx12G" - -help: - @echo "Try 'make Amsterdam'" - @echo "Docker must be installed" - @echo "'make list' for all available metro areas." - -list: - @echo ${CITIES} - -$(filter %,$(CITIES)): %: \ - ${DATA_DIR}/%.osm.pbf \ - ${DATA_DIR}/%.gtfs.csv \ - ${DATA_DIR}/%.gtfs.tar \ - ${DATA_DIR}/%.nominatim.sql.bz2 \ - ${DATA_DIR}/%.nominatim_tokenizer.tgz \ - ${DATA_DIR}/%.photon.tar.bz2 \ - ${DATA_DIR}/%.mbtiles \ - ${DATA_DIR}/%.graph.obj \ - ${DATA_DIR}/%.valhalla.tar.bz2 \ - ${DATA_DIR}/fonts.tar \ - ${DATA_DIR}/sprite.tar \ - tag_images - @echo "Built $@" - -%.custom: \ - ${DATA_DIR}/%.osm.pbf \ - ${DATA_DIR}/%.gtfs.csv \ - ${DATA_DIR}/%.gtfs.tar \ - ${DATA_DIR}/%.nominatim.sql.bz2 \ - ${DATA_DIR}/%.nominatim_tokenizer.tgz \ - ${DATA_DIR}/%.photon.tar.bz2 \ - ${DATA_DIR}/%.mbtiles \ - ${DATA_DIR}/%.graph.obj \ - ${DATA_DIR}/%.valhalla.tar.bz2 \ - ${DATA_DIR}/fonts.tar \ - ${DATA_DIR}/sprite.tar \ - tag_images - @echo "Built $@" - -%.osm.pbf: - mkdir -p ${DATA_DIR} - echo "Downloading $(notdir $*) from BBBike." - wget -U headway/1.0 -O $@ "https://download.bbbike.org/osm/bbbike/$(notdir $*)/$(notdir $@)" || rm $@ - @echo -e "\n\nConsider donating to BBBike to help cover hosting! https://extract.bbbike.org/community.html\n\n" - -%.gtfs.csv: - touch $@ - @echo "To build with transit, run 'make $(notdir $*)'.enumerate_gtfs_feeds and manually edit $@ to curate transit agencies." - @echo "Building without transit." - -%.enumerate_gtfs_feeds: - set -e ;\ - ITAG=headway_enumerate_gtfs_$$(echo $(notdir $*) | tr '[:upper:]' '[:lower:]') ;\ - HEADWAY_BBOX=$$(grep "$(notdir $*):" web/bboxes.csv | cut -d':' -f2) ;\ - docker build ./gtfs/enumerate --build-arg HEADWAY_BBOX="$${HEADWAY_BBOX}" --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/gtfs_feeds/gtfs_feeds.csv ${DATA_DIR}/$(notdir $*).gtfs.csv ;\ - docker rm -v $$CID - -%.gtfs.tar: %.gtfs.csv - cp ${DATA_DIR}/$(notdir $*).gtfs.csv gtfs/download/gtfs_feeds.gen.csv - set -e ;\ - ITAG=headway_download_gtfs_$$(echo $(notdir $*) | tr '[:upper:]' '[:lower:]') ;\ - docker build ./gtfs/download --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/gtfs_feeds/gtfs.tar $@ ;\ - docker rm -v $$CID \ - -%.nominatim.sql.bz2 %.nominatim_tokenizer.tgz: %.osm.pbf - @echo "Building geocoding index for $(basename $(basename $@))." - cp $^ ./geocoder/nominatim_build/data.osm.pbf - set -e ;\ - ITAG=headway_build_nominatim_$$(echo $(notdir $*) | tr '[:upper:]' '[:lower:]') ;\ - docker build ./geocoder/nominatim_build --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/dump/nominatim.sql.bz2 $*.nominatim.sql.bz2 ;\ - docker cp $$CID:/nominatim/tokenizer.tgz $*.nominatim_tokenizer.tgz ;\ - docker rm -v $$CID - -%.photon.tar.bz2: %.nominatim.sql.bz2 - @echo "Importing data into photon and building index for $*." - cp $^ ./geocoder/photon_build/data.nominatim.sql.bz2 - set -e ;\ - ITAG=headway_build_photon_$$(echo $(notdir $*) | tr '[:upper:]' '[:lower:]') ;\ - docker build ./geocoder/photon_build --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/photon/photon.tar.bz2 $@ ;\ - docker rm -v $$CID - -%.mbtiles: %.osm.pbf - @echo "Building MBTiles $*" - cp $*.osm.pbf mbtiles_build/data.osm.pbf - set -e ;\ - ITAG=headway_build_mbtiles_$$(echo $(notdir $*) | tr '[:upper:]' '[:lower:]') ;\ - docker build ./mbtiles_build --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/data/output.mbtiles $@ ;\ - docker rm -v $$CID - -%.graph.obj: %.osm.pbf %.gtfs.csv %.gtfs.tar - @echo "Building OpenTripPlanner graph for $*." - cp $*.osm.pbf ./otp/build/data.osm.pbf - cp $*.gtfs.tar ./otp/build/gtfs.tar - set -e ;\ - ITAG=headway_build_otp_$$(echo $(notdir $*) | tr '[:upper:]' '[:lower:]') ;\ - docker build ./otp/build --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/data/graph.obj $@ ;\ - docker rm -v $$CID - -%.valhalla.tar.bz2: %.osm.pbf - @echo "Building Valhalla tiles for $(basename $(basename $@))." - cp $< ./valhalla/build/data.osm.pbf - set -e ;\ - ITAG=headway_build_valhalla_$$(echo $(notdir $*) | tr '[:upper:]' '[:lower:]') ;\ - docker build ./valhalla/build --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/tiles/valhalla.tar.bz2 $@ ;\ - docker rm -v $$CID - -# fontnik only runs on amd64 node images unfortunately. -${DATA_DIR}/fonts.tar ${DATA_DIR}/sprite.tar: - set -e ;\ - ITAG=headway_build_fonts ;\ - docker build ./tileserver/assets --tag $${ITAG} ;\ - CID=$$(docker create $${ITAG}) ;\ - docker cp $$CID:/output/fonts.tar ${DATA_DIR}/fonts.tar ;\ - docker cp $$CID:/output/sprite.tar ${DATA_DIR}/sprite.tar ;\ - docker rm -v $$CID ;\ - mkdir -p ${DATA_DIR}/fonts && cd ${DATA_DIR}/fonts && tar xvf ../fonts.tar ;\ - mkdir -p ${DATA_DIR}/sprite && cd ${DATA_DIR}/sprite && tar xvf ../sprite.tar - -tag_images: nginx_image photon_image nominatim_image otp_image valhalla_image tileserver_image - @echo "Tagged images" - -nginx_image: - docker build ./web --tag headway_nginx - -photon_image: - @echo "Building photon image" - docker build ./geocoder/photon --tag headway_photon - -nominatim_image: - @echo "Building nominatim image" - docker build ./geocoder/nominatim --tag headway_nominatim - -otp_image: - docker build ./otp/run --tag headway_otp - -valhalla_image: - docker build ./valhalla/run --tag headway_valhalla - -# These copies may often be unnecessary, but they're cheap and better to do them too much than forget to do them. -tileserver_image: - mkdir -p ${DATA_DIR}/styles - cp tileserver/style/style.json.template ${DATA_DIR}/styles/bright.json.template - docker build ./tileserver/image --tag headway_tileserver - -%.up: % - docker-compose kill || echo "Containers not up" - docker-compose down || echo "Containers dont exist" - docker-compose up -d - -# Clean only generated data. -clean: - rm -rf ${DATA_DIR}/*.mbtiles - rm -rf ${DATA_DIR}/*.nominatim.sql - rm -rf ${DATA_DIR}/*.nominatim.sql.bz2 - rm -rf ${DATA_DIR}/*.nominatim_tokenizer.tgz - rm -rf ${DATA_DIR}/*.photon.tgz - rm -rf ${DATA_DIR}/*.photon.tar.bz2 - rm -rf ${DATA_DIR}/*.valhalla.tar - rm -rf ${DATA_DIR}/*.valhalla.tar.bz2 - rm -rf ${DATA_DIR}/*.graph.obj - rm -rf ${DATA_DIR}/fonts* - rm -rf ${DATA_DIR}/sprite* - rm -rf ${DATA_DIR}/styles - -# Clean even the data we have to download from external sources. -clean_all: clean - rm -rf ${DATA_DIR}/*.osm.pbf - rm -rf ${DATA_DIR}/*.gtfs.tar - rm -rf ${DATA_DIR}/*.gtfs.csv - rm -rf ${DATA_DIR}/sources - docker images -qf "reference=headway_build_*" --format='{{.Repository}}:{{.Tag}}' | xargs docker rmi diff --git a/docker-compose.yaml b/docker-compose.yaml index 561363bd3..12b54773b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,48 +1,121 @@ version: "3" services: + mbtileserver_init: + image: headway_tileserver_init + env_file: .env + environment: + FONT_ARTIFACT_SOURCE_PATH: /bootstrap/fonts.tar + FONT_ARTIFACT_DEST_PATH: /data/fonts/fonts.tar + SPRITE_ARTIFACT_SOURCE_PATH: /bootstrap/sprite.tar + SPRITE_ARTIFACT_DEST_PATH: /data/sprite/sprite.tar + MBTILES_ARTIFACT_DEST_PATH: /data/${HEADWAY_AREA}.mbtiles + MBTILES_ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.mbtiles + volumes: + - "./data/:/bootstrap/:ro" + - "mbtileserver_data:/data/:rw" mbtileserver: image: headway_tileserver - build: ./tileserver/image environment: PORT: 8000 env_file: .env volumes: - - "./data/:/data/:ro" + - "mbtileserver_data:/data/:ro" + depends_on: + mbtileserver_init: + condition: service_completed_successfully + photon_init: + image: headway_photon_init + env_file: .env + environment: + ARTIFACT_LOCK: /photon/photon_data/imported + ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.photon.tar.bz2 + volumes: + - "./data/:/bootstrap/:ro" + - "photon_data:/photon/photon_data:rw" photon: image: headway_photon - build: ./geocoder/photon env_file: .env volumes: - - "./data/:/data/:ro" + - "photon_data:/photon/photon_data:rw" + depends_on: + photon_init: + condition: service_completed_successfully + nominatim_init: + image: headway_nominatim_init + env_file: .env + environment: + ARTIFACT_LOCK: /photon/photon_data/imported + PBF_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.osm.pbf + PBF_PATH: /nominatim_extra/${HEADWAY_AREA}.osm.pbf + TOKENIZER_ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.tokenizer.tar + DUMP_ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.nominatim.sql.bz2 + volumes: + - "./data/:/bootstrap/:ro" + - "extra_vol:/nominatim_extra:rw" + - "nominatim_data:/var/lib/postgresql/12/main:rw" nominatim: image: headway_nominatim - build: ./geocoder/nominatim environment: - PBF_PATH: /data/${HEADWAY_AREA}.osm.pbf + PBF_PATH: /nominatim_extra/${HEADWAY_AREA}.osm.pbf HEADWAY_NOMINATIM_FILE: /data/${HEADWAY_AREA}.nominatim.sql.bz2 HEADWAY_NOMINATIM_TOKENIZER: /data/${HEADWAY_AREA}.nominatim_tokenizer.tgz env_file: .env volumes: - - "./data/:/data/:ro" + - "extra_vol:/nominatim_extra:ro" + - "nominatim_data:/var/lib/postgresql/12/main:rw" + depends_on: + nominatim_init: + condition: service_completed_successfully + otp_init: + image: headway_otp_init + env_file: .env + environment: + ARTIFACT_DEST_PATH: /data/${HEADWAY_AREA}.graph.obj + ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.graph.obj + volumes: + - "./data/:/bootstrap/:ro" + - "otp_data:/data/:rw" otp: image: headway_otp - build: ./otp/run env_file: .env volumes: - - "./data/:/data/:ro" + - "otp_data:/data/:ro" + depends_on: + otp_init: + condition: service_completed_successfully + valhalla_init: + image: headway_valhalla_init + env_file: .env + volumes: + - "valhalla_data:/data/:rw" + - "./data/:/bootstrap/:ro" + environment: + ARTIFACT_SOURCE_PATH: /bootstrap/${HEADWAY_AREA}.valhalla.tar.bz2 + ARTIFACT_LOCK: /data/imported + ulimits: + nofile: + soft: 8192 + hard: 8192 valhalla: image: headway_valhalla - build: ./valhalla/run env_file: .env volumes: - - "./data/:/data_mount/:ro" + - "valhalla_data:/data/:ro" ulimits: nofile: soft: 8192 hard: 8192 + depends_on: + valhalla_init: + condition: service_completed_successfully + nginx_init: + image: headway_nginx_init + env_file: .env + volumes: + - "./data/:/bootstrap/:ro" + - "nginx_data:/data/:rw" nginx: image: headway_nginx - build: ./web env_file: .env environment: HEADWAY_RESOLVER: 127.0.0.11 @@ -61,3 +134,11 @@ services: - "nominatim" - "photon" - "mbtileserver" +volumes: + extra_vol: + photon_data: + nominatim_data: + mbtileserver_data: + otp_data: + valhalla_data: + nginx_data: diff --git a/geocoder/nominatim/Dockerfile b/geocoder/nominatim/Dockerfile deleted file mode 100644 index 529e1ef05..000000000 --- a/geocoder/nominatim/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM mediagis/nominatim:4.0 - -RUN apt-get update -y && apt-get install -y pbzip2 - -COPY restore_then_start.sh /app - -ENTRYPOINT [ "/bin/bash" ] - -CMD [ "restore_then_start.sh" ] diff --git a/geocoder/nominatim/restore_then_start.sh b/geocoder/nominatim/restore_then_start.sh deleted file mode 100644 index 7b3d7d18c..000000000 --- a/geocoder/nominatim/restore_then_start.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -xe - -(cd ${PROJECT_DIR} && tar xvf ${HEADWAY_NOMINATIM_TOKENIZER}) - -service postgresql start && \ -sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ -sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ - -# Maybe change this password? It really doesn't matter that much though, it's just ephemeral data. -sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD 'password1'" && \ -sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD 'password1'" && \ - -sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" - -sudo -E -u postgres psql postgres -c "CREATE DATABASE nominatim" - -echo "Beginning nominatim restore" -time cat ${HEADWAY_NOMINATIM_FILE} | pbzip2 -d | sudo -E -u postgres psql nominatim - -touch /var/lib/postgresql/12/main/import-finished - -/app/start.sh diff --git a/geocoder/nominatim_build/.gitignore b/geocoder/nominatim_build/.gitignore deleted file mode 100644 index 11f08d094..000000000 --- a/geocoder/nominatim_build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.osm.pbf diff --git a/geocoder/nominatim_build/Dockerfile b/geocoder/nominatim_build/Dockerfile deleted file mode 100644 index 74b56d123..000000000 --- a/geocoder/nominatim_build/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM mediagis/nominatim:4.0 - -RUN apt-get update -y && apt-get install -y pbzip2 - -COPY data.osm.pbf /data.osm.pbf - -RUN bash -c 'useradd -m -p ${NOMINATIM_PASSWORD} nominatim' - -ENV PBF_PATH=/data.osm.pbf - -RUN bash -c '/app/config.sh' - -RUN bash -c '/app/init.sh' - -RUN touch /var/lib/postgresql/12/main/import-finished - -COPY import_then_dump.sh /app - -RUN /app/import_then_dump.sh diff --git a/geocoder/photon/.gitignore b/geocoder/photon/.gitignore deleted file mode 100644 index df8fb938b..000000000 --- a/geocoder/photon/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.nominatim.tgz \ No newline at end of file diff --git a/geocoder/photon/Dockerfile b/geocoder/photon/Dockerfile deleted file mode 100644 index fdcf78473..000000000 --- a/geocoder/photon/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM debian:bullseye-slim - -ENV TZ="America/New_York" - -RUN apt-get update \ - && apt-get install -y --no-install-recommends postgresql postgis postgresql-postgis curl openjdk-11-jre-headless sudo pbzip2 - -RUN mkdir /photon - -WORKDIR /photon - -RUN useradd -ms /bin/bash photon - -RUN chown photon /photon - -ARG PHOTON_VERSION=0.3.5 -ARG PHOTON_HASH=6f438e905b020a70ec54de47b7c7f8b9c7921ef0ee84a3a9e0a6a6ebb7f74fa8 - -RUN curl -L -o ./photon.jar https://github.com/komoot/photon/releases/download/${PHOTON_VERSION}/photon-${PHOTON_VERSION}.jar - -RUN echo "${PHOTON_HASH} ./photon.jar" | sha256sum --check - -COPY untar_and_start.sh /photon - -CMD [ "/photon/untar_and_start.sh" ] diff --git a/geocoder/photon/untar_and_start.sh b/geocoder/photon/untar_and_start.sh deleted file mode 100755 index 0ee63d08b..000000000 --- a/geocoder/photon/untar_and_start.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -xe - -echo "Extracting photon index" - -cd / && time cat /data/${HEADWAY_AREA}.photon.tar.bz2 | pbzip2 -d | tar x - -echo "Starting photon" - -cd /photon && sudo -E -u photon java -jar /photon/photon.jar diff --git a/geocoder/photon_build/.gitignore b/geocoder/photon_build/.gitignore deleted file mode 100644 index 11f08d094..000000000 --- a/geocoder/photon_build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.osm.pbf diff --git a/geocoder/photon_build/Dockerfile b/geocoder/photon_build/Dockerfile deleted file mode 100644 index e0e70b563..000000000 --- a/geocoder/photon_build/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM mediagis/nominatim:4.0 - -ENV TZ="America/New_York" - -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl openjdk-11-jre-headless sudo pbzip2 - -RUN mkdir /photon - -WORKDIR /photon - -RUN useradd -ms /bin/bash photon - -RUN chown photon /photon - -ARG PHOTON_VERSION=0.3.5 -ARG PHOTON_HASH=6f438e905b020a70ec54de47b7c7f8b9c7921ef0ee84a3a9e0a6a6ebb7f74fa8 - -RUN curl -L -o ./photon.jar https://github.com/komoot/photon/releases/download/${PHOTON_VERSION}/photon-${PHOTON_VERSION}.jar - -RUN echo "${PHOTON_HASH} ./photon.jar" | sha256sum --check - -COPY import_from_dump.sh /photon - -USER root - -COPY data.nominatim.sql.bz2 /nominatim_data/data.nominatim.sql.bz2 - -WORKDIR /photon - -RUN ["/bin/bash", "/photon/import_from_dump.sh"] - -USER photon - -CMD [ "java", "-jar", "photon.jar" ] diff --git a/gtfs/download/.gitignore b/gtfs/download/.gitignore deleted file mode 100644 index afed0735d..000000000 --- a/gtfs/download/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.csv diff --git a/gtfs/download/Dockerfile b/gtfs/download/Dockerfile deleted file mode 100644 index 28b02f811..000000000 --- a/gtfs/download/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3 - -RUN pip install requests - -WORKDIR /gtfs - -RUN mkdir /gtfs_feeds - -COPY download_gtfs_feeds.py /gtfs/ -COPY gtfs_feeds.gen.csv /gtfs/gtfs_feeds.csv - -RUN python /gtfs/download_gtfs_feeds.py - -RUN bash -c "cd /gtfs_feeds && ls *.zip | tar -cf gtfs.tar --files-from -" diff --git a/gtfs/enumerate/Dockerfile b/gtfs/enumerate/Dockerfile deleted file mode 100644 index 9104812c3..000000000 --- a/gtfs/enumerate/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3 - -RUN pip install requests - -WORKDIR /gtfs - -RUN mkdir /gtfs_feeds - -COPY enumerate_gtfs_feeds.py /gtfs/ - -ARG HEADWAY_BBOX - -ENV HEADWAY_BBOX=${HEADWAY_BBOX} - -RUN python /gtfs/enumerate_gtfs_feeds.py diff --git a/mbtiles_build/.gitignore b/mbtiles_build/.gitignore deleted file mode 100644 index 4a0118a9d..000000000 --- a/mbtiles_build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.osm.pbf \ No newline at end of file diff --git a/mbtiles_build/Dockerfile b/mbtiles_build/Dockerfile deleted file mode 100644 index b8f88a7e8..000000000 --- a/mbtiles_build/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM debian:bullseye-slim as bootstrapper - -RUN apt-get update -y && apt-get install -y --no-install-recommends wget ca-certificates - -COPY download_mirrored_data.sh / - -RUN /download_mirrored_data.sh - -FROM debian:bullseye-slim - -RUN apt-get update -y && apt-get install -y --no-install-recommends curl ca-certificates openjdk-17-jre-headless - -WORKDIR /planetiler - -ARG PLANETILER_VERSION=v0.4.0 -ARG PLANETILER_HASH=2ec77024662909fc858a6aae16973f3f15a26429ccd88f06d5998ab4241108e0 - -ARG JAVA_MEM_ARGS=-Xmx8G - -RUN curl -L -o planetiler.jar https://github.com/onthegomap/planetiler/releases/download/${PLANETILER_VERSION}/planetiler.jar - -RUN bash -c 'echo "${PLANETILER_HASH} planetiler.jar" | sha256sum --check' - -COPY --from=bootstrapper /data/lake_centerline.shp.zip /data/natural_earth_vector.sqlite.zip /data/water-polygons-split-3857.zip /data/sources/ - -COPY data.osm.pbf /data/data.osm.pbf - -WORKDIR / - -RUN java ${JAVA_MEM_ARGS} -jar /planetiler/planetiler.jar --force osm_path=/data/data.osm.pbf diff --git a/mbtiles_build/download_mirrored_data.sh b/mbtiles_build/download_mirrored_data.sh deleted file mode 100755 index e19dfc217..000000000 --- a/mbtiles_build/download_mirrored_data.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -mkdir -p /data -cd /data - -rm -f sources.tar -ls *.zip || wget https://f000.backblazeb2.com/file/headway/sources.tar && tar xvf sources.tar -rm -f sources.tar diff --git a/otp/build/.gitignore b/otp/build/.gitignore deleted file mode 100644 index 83d615d90..000000000 --- a/otp/build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -gtfs.tar diff --git a/otp/build/Dockerfile b/otp/build/Dockerfile deleted file mode 100644 index 56442d519..000000000 --- a/otp/build/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM debian:bullseye-slim - -ENV TZ="America/New_York" - -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl openjdk-11-jre-headless sudo - -ARG OTP_VERISON=2.1.0 -ARG OTP_HASH=b4c986b1c726c7d81d255fa183d32576122ba4e50290d53e4bb40be051971134 - -WORKDIR /otp - -RUN curl -L -o ./otp-shaded.jar "https://github.com/opentripplanner/OpenTripPlanner/releases/download/v${OTP_VERISON}/otp-${OTP_VERISON}-shaded.jar" - -RUN bash -c 'echo "${OTP_HASH} otp-shaded.jar" | sha256sum --check' - -RUN mkdir /data - -COPY gtfs.tar data.osm.pbf /data/ - -WORKDIR /data - -RUN tar xvf gtfs.tar - -ARG JAVA_MEM_ARGS=-Xmx4G - -COPY build-config.json /data - -COPY maybe_build.sh /otp - -RUN /otp/maybe_build.sh diff --git a/otp/build/build-config.json b/otp/build/build-config.json deleted file mode 100644 index 8473dd65d..000000000 --- a/otp/build/build-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "transitServiceStart": "-P10Y" -} diff --git a/otp/run/Dockerfile b/otp/run/Dockerfile deleted file mode 100644 index f9d92c3e2..000000000 --- a/otp/run/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM debian:bullseye-slim - -ENV TZ="America/New_York" - -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl openjdk-11-jre-headless sudo netcat - -ARG OTP_VERISON=2.1.0 -ARG OTP_HASH=b4c986b1c726c7d81d255fa183d32576122ba4e50290d53e4bb40be051971134 - -WORKDIR /otp - -RUN curl -L -o ./otp-shaded.jar "https://github.com/opentripplanner/OpenTripPlanner/releases/download/v${OTP_VERISON}/otp-${OTP_VERISON}-shaded.jar" - -RUN bash -c 'echo "${OTP_HASH} otp-shaded.jar" | sha256sum --check' - -ENV JAVA_MEM_ARGS=-Xmx4G - -COPY rename_and_run.sh /otp - -CMD [ "/otp/rename_and_run.sh" ] diff --git a/gtfs/download/download_gtfs_feeds.py b/services/gtfs/download_gtfs_feeds.py similarity index 100% rename from gtfs/download/download_gtfs_feeds.py rename to services/gtfs/download_gtfs_feeds.py diff --git a/gtfs/enumerate/enumerate_gtfs_feeds.py b/services/gtfs/enumerate_gtfs_feeds.py similarity index 100% rename from gtfs/enumerate/enumerate_gtfs_feeds.py rename to services/gtfs/enumerate_gtfs_feeds.py diff --git a/services/init_container_dl.sh b/services/init_container_dl.sh new file mode 100644 index 000000000..f618e58d5 --- /dev/null +++ b/services/init_container_dl.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ -f "${ARTIFACT_DEST_PATH}" ] + echo "Nothing to do, already have ${ARTIFACT_DEST_PATH}" + exit 0 +fi + +if [ -f "${ARTIFACT_SOURCE_PATH}" ] + echo "Creating symbolic link to artifact location." + cp "${ARTIFACT_SOURCE_PATH}" "${ARTIFACT_DEST_PATH}" + exit 0 +fi + +echo "Downloading artifact" +wget -O "${ARTIFACT_DEST_PATH}" "${ARTIFACT_URL}" diff --git a/services/nginx/init.sh b/services/nginx/init.sh new file mode 100755 index 000000000..b5ea79de8 --- /dev/null +++ b/services/nginx/init.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +if [ -f "${HEADWAY_BBOX_PATH}" ]; then + echo "Nothing to do, already have ${HEADWAY_BBOX_PATH}" + exit 0 +fi + +if [ -f "${BBOX_SOURCE_PATH}" ]; then + echo "Copying artifact." + cp "${BBOX_SOURCE_PATH}" "${HEADWAY_BBOX_PATH}" + exit 0 +fi + +echo "Downloading artifact" +wget -O "${ARTIFACT_DEST_PATH}" "${ARTIFACT_URL}" diff --git a/geocoder/nominatim_build/import_then_dump.sh b/services/nominatim/dump.sh similarity index 66% rename from geocoder/nominatim_build/import_then_dump.sh rename to services/nominatim/dump.sh index 5e4021f46..f65043a82 100755 --- a/geocoder/nominatim_build/import_then_dump.sh +++ b/services/nominatim/dump.sh @@ -1,5 +1,3 @@ -#!/bin/bash - set -xe service postgresql start @@ -8,4 +6,4 @@ mkdir -p /dump sudo -u nominatim pg_dump nominatim | pbzip2 -c -6 > /dump/nominatim.sql.bz2 -cd ${PROJECT_DIR} && tar czf tokenizer.tgz tokenizer +cd ${PROJECT_DIR} && tar cf tokenizer.tar tokenizer diff --git a/services/nominatim/init.sh b/services/nominatim/init.sh new file mode 100755 index 000000000..197f0f94b --- /dev/null +++ b/services/nominatim/init.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# To be used for k8s init containers or run prior to each startup in docker-compose. + +set -xe + +if [ -f "/var/lib/postgresql/12/main/import-finished" ]; then + echo "Nothing to do, already imported." + exit 0 +fi + +if [ -f "${PBF_SOURCE_PATH}" ]; then + echo "Copying artifact." + cp "${PBF_SOURCE_PATH}" "${PBF_PATH}" +else + echo "Downloading artifact" + cd / && wget -O "${PBF_PATH}" "${PBF_URL}" +fi + +if [ -f "${TOKENIZER_ARTIFACT_SOURCE_PATH}" ]; then + echo "Copying artifact." + cp "${TOKENIZER_ARTIFACT_SOURCE_PATH}" /nominatim_extra/tokenizer.tar +else + echo "Downloading artifact" + cd / && wget -O /nominatim_extra/tokenizer.tar "${TOKENIZER_ARTIFACT_URL}" +fi + +service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +# Ephemeral data from a public dataset. Password can be whatever. +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD 'password1'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD 'password1'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +sudo -E -u postgres psql postgres -c "CREATE DATABASE nominatim" + +echo "Beginning nominatim restore" + +if [ -f "${DUMP_ARTIFACT_SOURCE_PATH}" ]; then + cd / && cat "${DUMP_ARTIFACT_SOURCE_PATH}" | pbzip2 -d | sudo -E -u postgres psql nominatim +else + cd / && wget -qO- "${DUMP_ARTIFACT_URL}" | pbzip2 -d | sudo -E -u postgres psql nominatim +fi + +service postgresql stop + +touch /var/lib/postgresql/12/main/import-finished + +# At this point nominatim is ready for a warm start :) diff --git a/services/nominatim/run.sh b/services/nominatim/run.sh new file mode 100755 index 000000000..b31b32452 --- /dev/null +++ b/services/nominatim/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -xe + +(cd ${PROJECT_DIR} && tar xvf /nominatim_extra/tokenizer.tar) + +/app/start.sh diff --git a/services/otp/init.sh b/services/otp/init.sh new file mode 100755 index 000000000..d832c3526 --- /dev/null +++ b/services/otp/init.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +if [ -f "${ARTIFACT_DEST_PATH}" ]; then + echo "Nothing to do, already have ${ARTIFACT_DEST_PATH}" + exit 0 +fi + +if [ -f "${ARTIFACT_SOURCE_PATH}" ]; then + echo "Copying artifact." + cp "${ARTIFACT_SOURCE_PATH}" "${ARTIFACT_DEST_PATH}" + exit 0 +fi + +echo "Downloading artifact" +wget -O "${ARTIFACT_DEST_PATH}" "${ARTIFACT_URL}" diff --git a/otp/build/maybe_build.sh b/services/otp/maybe_build.sh similarity index 100% rename from otp/build/maybe_build.sh rename to services/otp/maybe_build.sh diff --git a/otp/run/rename_and_run.sh b/services/otp/run_otp.sh similarity index 67% rename from otp/run/rename_and_run.sh rename to services/otp/run_otp.sh index e71c3a963..969c8b511 100755 --- a/otp/run/rename_and_run.sh +++ b/services/otp/run_otp.sh @@ -5,7 +5,8 @@ set -xe if [[ -s /data/${HEADWAY_AREA}.graph.obj ]]; then netcat -l -p 9999 # Hack: Signal to clients that we have a transit graph. - cp /data/${HEADWAY_AREA}.graph.obj /otp/graph.obj + echo "{\"transitServiceStart\": \"-P10Y\", \"streetGraph\": \"/data/${HEADWAY_AREA}.graph.obj\"}" > /otp/build-config.json + java ${JAVA_MEM_ARGS} -jar otp-shaded.jar --load /otp else while : diff --git a/geocoder/photon_build/import_from_dump.sh b/services/photon/import_and_dump.sh similarity index 72% rename from geocoder/photon_build/import_from_dump.sh rename to services/photon/import_and_dump.sh index 76a3d9b71..d8771a689 100755 --- a/geocoder/photon_build/import_from_dump.sh +++ b/services/photon/import_and_dump.sh @@ -1,15 +1,16 @@ #!/bin/bash +# Import an index from a nominatim dump, then persist the photon index data for later use. + set -xe HEADWAY_NOMINATIM_FILE=${HEADWAY_NOMINATIM_FILE:-/nominatim_data/data.nominatim.sql.bz2} -HEADWAY_PHOTON_LANGUAGES=${HEADWAY_PHOTON_LANGUAGES:-es,fr,de,en} service postgresql start && \ sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ -# Maybe change this password? It really doesn't matter that much though, it's just ephemeral data. +# Ephemeral data from a public dataset. Password can be whatever. sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD 'password1'" && \ sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD 'password1'" && \ @@ -19,7 +20,7 @@ sudo -E -u postgres psql postgres -c "CREATE DATABASE nominatim" cat ${HEADWAY_NOMINATIM_FILE} | pbzip2 -d | sudo -E -u postgres psql nominatim -sudo -u photon /bin/sh -c "java -jar /photon/photon.jar -nominatim-import -host localhost -port 5432 -database nominatim -user nominatim -password password1 -languages ${HEADWAY_PHOTON_LANGUAGES}" +sudo -u photon /bin/sh -c "cd /photon && java -jar /photon/photon.jar -nominatim-import -host localhost -port 5432 -database nominatim -user nominatim -password password1 -languages ${HEADWAY_PHOTON_LANGUAGES}" mkdir -p /dump diff --git a/services/photon/init.sh b/services/photon/init.sh new file mode 100755 index 000000000..aaee64172 --- /dev/null +++ b/services/photon/init.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -xe + +if [ -f "${ARTIFACT_LOCK}" ]; then + echo "Nothing to do, already have artifact." + exit 0 +fi + +if [ -f "${ARTIFACT_SOURCE_PATH}" ]; then + echo "Copying artifact." + cd / && cat "${ARTIFACT_SOURCE_PATH}" | pbzip2 -d | tar x +else + echo "Downloading artifact" + cd / && wget -qO- "${ARTIFACT_URL}" | pbzip2 -d | tar x +fi + +chown -R photon /photon + +# Don't do this again next time. +touch ${ARTIFACT_LOCK} + +ls -lR /photon + +# At this point Photon is ready for a warm start. :) diff --git a/tileserver/assets/.dockerignore b/services/tileserver/assets/.dockerignore similarity index 100% rename from tileserver/assets/.dockerignore rename to services/tileserver/assets/.dockerignore diff --git a/tileserver/assets/.gitignore b/services/tileserver/assets/.gitignore similarity index 100% rename from tileserver/assets/.gitignore rename to services/tileserver/assets/.gitignore diff --git a/tileserver/assets/COPYRIGHT_NOTICE.md b/services/tileserver/assets/COPYRIGHT_NOTICE.md similarity index 100% rename from tileserver/assets/COPYRIGHT_NOTICE.md rename to services/tileserver/assets/COPYRIGHT_NOTICE.md diff --git a/tileserver/assets/Dockerfile b/services/tileserver/assets/Dockerfile similarity index 100% rename from tileserver/assets/Dockerfile rename to services/tileserver/assets/Dockerfile diff --git a/tileserver/assets/NotoSans-Bold.ttf b/services/tileserver/assets/NotoSans-Bold.ttf similarity index 100% rename from tileserver/assets/NotoSans-Bold.ttf rename to services/tileserver/assets/NotoSans-Bold.ttf diff --git a/tileserver/assets/NotoSans-Italic.ttf b/services/tileserver/assets/NotoSans-Italic.ttf similarity index 100% rename from tileserver/assets/NotoSans-Italic.ttf rename to services/tileserver/assets/NotoSans-Italic.ttf diff --git a/tileserver/assets/NotoSans-Regular.ttf b/services/tileserver/assets/NotoSans-Regular.ttf similarity index 100% rename from tileserver/assets/NotoSans-Regular.ttf rename to services/tileserver/assets/NotoSans-Regular.ttf diff --git a/tileserver/assets/OFL.txt b/services/tileserver/assets/OFL.txt similarity index 100% rename from tileserver/assets/OFL.txt rename to services/tileserver/assets/OFL.txt diff --git a/tileserver/assets/build_glyphs.js b/services/tileserver/assets/build_glyphs.js similarity index 100% rename from tileserver/assets/build_glyphs.js rename to services/tileserver/assets/build_glyphs.js diff --git a/tileserver/assets/build_sprites.js b/services/tileserver/assets/build_sprites.js similarity index 100% rename from tileserver/assets/build_sprites.js rename to services/tileserver/assets/build_sprites.js diff --git a/tileserver/assets/package-lock.json b/services/tileserver/assets/package-lock.json similarity index 100% rename from tileserver/assets/package-lock.json rename to services/tileserver/assets/package-lock.json diff --git a/tileserver/assets/package.json b/services/tileserver/assets/package.json similarity index 100% rename from tileserver/assets/package.json rename to services/tileserver/assets/package.json diff --git a/tileserver/assets/sprites/LICENSE.md b/services/tileserver/assets/sprites/LICENSE.md similarity index 100% rename from tileserver/assets/sprites/LICENSE.md rename to services/tileserver/assets/sprites/LICENSE.md diff --git a/tileserver/assets/sprites/airfield_11.svg b/services/tileserver/assets/sprites/airfield_11.svg similarity index 100% rename from tileserver/assets/sprites/airfield_11.svg rename to services/tileserver/assets/sprites/airfield_11.svg diff --git a/tileserver/assets/sprites/airport_11.svg b/services/tileserver/assets/sprites/airport_11.svg similarity index 100% rename from tileserver/assets/sprites/airport_11.svg rename to services/tileserver/assets/sprites/airport_11.svg diff --git a/tileserver/assets/sprites/alcohol_shop_11.svg b/services/tileserver/assets/sprites/alcohol_shop_11.svg similarity index 100% rename from tileserver/assets/sprites/alcohol_shop_11.svg rename to services/tileserver/assets/sprites/alcohol_shop_11.svg diff --git a/tileserver/assets/sprites/amusement_park_11.svg b/services/tileserver/assets/sprites/amusement_park_11.svg similarity index 100% rename from tileserver/assets/sprites/amusement_park_11.svg rename to services/tileserver/assets/sprites/amusement_park_11.svg diff --git a/tileserver/assets/sprites/aquarium_11.svg b/services/tileserver/assets/sprites/aquarium_11.svg similarity index 100% rename from tileserver/assets/sprites/aquarium_11.svg rename to services/tileserver/assets/sprites/aquarium_11.svg diff --git a/tileserver/assets/sprites/art_gallery_11.svg b/services/tileserver/assets/sprites/art_gallery_11.svg similarity index 100% rename from tileserver/assets/sprites/art_gallery_11.svg rename to services/tileserver/assets/sprites/art_gallery_11.svg diff --git a/tileserver/assets/sprites/attraction_11.svg b/services/tileserver/assets/sprites/attraction_11.svg similarity index 100% rename from tileserver/assets/sprites/attraction_11.svg rename to services/tileserver/assets/sprites/attraction_11.svg diff --git a/tileserver/assets/sprites/bakery_11.svg b/services/tileserver/assets/sprites/bakery_11.svg similarity index 100% rename from tileserver/assets/sprites/bakery_11.svg rename to services/tileserver/assets/sprites/bakery_11.svg diff --git a/tileserver/assets/sprites/bank_11.svg b/services/tileserver/assets/sprites/bank_11.svg similarity index 100% rename from tileserver/assets/sprites/bank_11.svg rename to services/tileserver/assets/sprites/bank_11.svg diff --git a/tileserver/assets/sprites/bar_11.svg b/services/tileserver/assets/sprites/bar_11.svg similarity index 100% rename from tileserver/assets/sprites/bar_11.svg rename to services/tileserver/assets/sprites/bar_11.svg diff --git a/tileserver/assets/sprites/beer_11.svg b/services/tileserver/assets/sprites/beer_11.svg similarity index 100% rename from tileserver/assets/sprites/beer_11.svg rename to services/tileserver/assets/sprites/beer_11.svg diff --git a/tileserver/assets/sprites/bicycle_11.svg b/services/tileserver/assets/sprites/bicycle_11.svg similarity index 100% rename from tileserver/assets/sprites/bicycle_11.svg rename to services/tileserver/assets/sprites/bicycle_11.svg diff --git a/tileserver/assets/sprites/bicycle_rental_11.svg b/services/tileserver/assets/sprites/bicycle_rental_11.svg similarity index 100% rename from tileserver/assets/sprites/bicycle_rental_11.svg rename to services/tileserver/assets/sprites/bicycle_rental_11.svg diff --git a/tileserver/assets/sprites/bus_11.svg b/services/tileserver/assets/sprites/bus_11.svg similarity index 100% rename from tileserver/assets/sprites/bus_11.svg rename to services/tileserver/assets/sprites/bus_11.svg diff --git a/tileserver/assets/sprites/cafe_11.svg b/services/tileserver/assets/sprites/cafe_11.svg similarity index 100% rename from tileserver/assets/sprites/cafe_11.svg rename to services/tileserver/assets/sprites/cafe_11.svg diff --git a/tileserver/assets/sprites/campsite_11.svg b/services/tileserver/assets/sprites/campsite_11.svg similarity index 100% rename from tileserver/assets/sprites/campsite_11.svg rename to services/tileserver/assets/sprites/campsite_11.svg diff --git a/tileserver/assets/sprites/car_11.svg b/services/tileserver/assets/sprites/car_11.svg similarity index 100% rename from tileserver/assets/sprites/car_11.svg rename to services/tileserver/assets/sprites/car_11.svg diff --git a/tileserver/assets/sprites/castle_11.svg b/services/tileserver/assets/sprites/castle_11.svg similarity index 100% rename from tileserver/assets/sprites/castle_11.svg rename to services/tileserver/assets/sprites/castle_11.svg diff --git a/tileserver/assets/sprites/cemetery_11.svg b/services/tileserver/assets/sprites/cemetery_11.svg similarity index 100% rename from tileserver/assets/sprites/cemetery_11.svg rename to services/tileserver/assets/sprites/cemetery_11.svg diff --git a/tileserver/assets/sprites/cinema_11.svg b/services/tileserver/assets/sprites/cinema_11.svg similarity index 100% rename from tileserver/assets/sprites/cinema_11.svg rename to services/tileserver/assets/sprites/cinema_11.svg diff --git a/tileserver/assets/sprites/circle_11.svg b/services/tileserver/assets/sprites/circle_11.svg similarity index 100% rename from tileserver/assets/sprites/circle_11.svg rename to services/tileserver/assets/sprites/circle_11.svg diff --git a/tileserver/assets/sprites/circle_stroked_11.svg b/services/tileserver/assets/sprites/circle_stroked_11.svg similarity index 100% rename from tileserver/assets/sprites/circle_stroked_11.svg rename to services/tileserver/assets/sprites/circle_stroked_11.svg diff --git a/tileserver/assets/sprites/clothing_store_11.svg b/services/tileserver/assets/sprites/clothing_store_11.svg similarity index 100% rename from tileserver/assets/sprites/clothing_store_11.svg rename to services/tileserver/assets/sprites/clothing_store_11.svg diff --git a/tileserver/assets/sprites/college_11.svg b/services/tileserver/assets/sprites/college_11.svg similarity index 100% rename from tileserver/assets/sprites/college_11.svg rename to services/tileserver/assets/sprites/college_11.svg diff --git a/tileserver/assets/sprites/dentist_11.svg b/services/tileserver/assets/sprites/dentist_11.svg similarity index 100% rename from tileserver/assets/sprites/dentist_11.svg rename to services/tileserver/assets/sprites/dentist_11.svg diff --git a/tileserver/assets/sprites/doctor_11.svg b/services/tileserver/assets/sprites/doctor_11.svg similarity index 100% rename from tileserver/assets/sprites/doctor_11.svg rename to services/tileserver/assets/sprites/doctor_11.svg diff --git a/tileserver/assets/sprites/dog_park_11.svg b/services/tileserver/assets/sprites/dog_park_11.svg similarity index 100% rename from tileserver/assets/sprites/dog_park_11.svg rename to services/tileserver/assets/sprites/dog_park_11.svg diff --git a/tileserver/assets/sprites/drinking_water_11.svg b/services/tileserver/assets/sprites/drinking_water_11.svg similarity index 100% rename from tileserver/assets/sprites/drinking_water_11.svg rename to services/tileserver/assets/sprites/drinking_water_11.svg diff --git a/tileserver/assets/sprites/embassy_11.svg b/services/tileserver/assets/sprites/embassy_11.svg similarity index 100% rename from tileserver/assets/sprites/embassy_11.svg rename to services/tileserver/assets/sprites/embassy_11.svg diff --git a/tileserver/assets/sprites/entrance_11.svg b/services/tileserver/assets/sprites/entrance_11.svg similarity index 100% rename from tileserver/assets/sprites/entrance_11.svg rename to services/tileserver/assets/sprites/entrance_11.svg diff --git a/tileserver/assets/sprites/fast_food_11.svg b/services/tileserver/assets/sprites/fast_food_11.svg similarity index 100% rename from tileserver/assets/sprites/fast_food_11.svg rename to services/tileserver/assets/sprites/fast_food_11.svg diff --git a/tileserver/assets/sprites/ferry_terminal_11.svg b/services/tileserver/assets/sprites/ferry_terminal_11.svg similarity index 100% rename from tileserver/assets/sprites/ferry_terminal_11.svg rename to services/tileserver/assets/sprites/ferry_terminal_11.svg diff --git a/tileserver/assets/sprites/fire_station_11.svg b/services/tileserver/assets/sprites/fire_station_11.svg similarity index 100% rename from tileserver/assets/sprites/fire_station_11.svg rename to services/tileserver/assets/sprites/fire_station_11.svg diff --git a/tileserver/assets/sprites/fuel_11.svg b/services/tileserver/assets/sprites/fuel_11.svg similarity index 100% rename from tileserver/assets/sprites/fuel_11.svg rename to services/tileserver/assets/sprites/fuel_11.svg diff --git a/tileserver/assets/sprites/garden_11.svg b/services/tileserver/assets/sprites/garden_11.svg similarity index 100% rename from tileserver/assets/sprites/garden_11.svg rename to services/tileserver/assets/sprites/garden_11.svg diff --git a/tileserver/assets/sprites/golf_11.svg b/services/tileserver/assets/sprites/golf_11.svg similarity index 100% rename from tileserver/assets/sprites/golf_11.svg rename to services/tileserver/assets/sprites/golf_11.svg diff --git a/tileserver/assets/sprites/grocery_11.svg b/services/tileserver/assets/sprites/grocery_11.svg similarity index 100% rename from tileserver/assets/sprites/grocery_11.svg rename to services/tileserver/assets/sprites/grocery_11.svg diff --git a/tileserver/assets/sprites/harbor_11.svg b/services/tileserver/assets/sprites/harbor_11.svg similarity index 100% rename from tileserver/assets/sprites/harbor_11.svg rename to services/tileserver/assets/sprites/harbor_11.svg diff --git a/tileserver/assets/sprites/heliport_11.svg b/services/tileserver/assets/sprites/heliport_11.svg similarity index 100% rename from tileserver/assets/sprites/heliport_11.svg rename to services/tileserver/assets/sprites/heliport_11.svg diff --git a/tileserver/assets/sprites/hospital_11.svg b/services/tileserver/assets/sprites/hospital_11.svg similarity index 100% rename from tileserver/assets/sprites/hospital_11.svg rename to services/tileserver/assets/sprites/hospital_11.svg diff --git a/tileserver/assets/sprites/ice_cream_11.svg b/services/tileserver/assets/sprites/ice_cream_11.svg similarity index 100% rename from tileserver/assets/sprites/ice_cream_11.svg rename to services/tileserver/assets/sprites/ice_cream_11.svg diff --git a/tileserver/assets/sprites/information_11.svg b/services/tileserver/assets/sprites/information_11.svg similarity index 100% rename from tileserver/assets/sprites/information_11.svg rename to services/tileserver/assets/sprites/information_11.svg diff --git a/tileserver/assets/sprites/laundry_11.svg b/services/tileserver/assets/sprites/laundry_11.svg similarity index 100% rename from tileserver/assets/sprites/laundry_11.svg rename to services/tileserver/assets/sprites/laundry_11.svg diff --git a/tileserver/assets/sprites/library_11.svg b/services/tileserver/assets/sprites/library_11.svg similarity index 100% rename from tileserver/assets/sprites/library_11.svg rename to services/tileserver/assets/sprites/library_11.svg diff --git a/tileserver/assets/sprites/lodging_11.svg b/services/tileserver/assets/sprites/lodging_11.svg similarity index 100% rename from tileserver/assets/sprites/lodging_11.svg rename to services/tileserver/assets/sprites/lodging_11.svg diff --git a/tileserver/assets/sprites/marker_11.svg b/services/tileserver/assets/sprites/marker_11.svg similarity index 100% rename from tileserver/assets/sprites/marker_11.svg rename to services/tileserver/assets/sprites/marker_11.svg diff --git a/tileserver/assets/sprites/monument_11.svg b/services/tileserver/assets/sprites/monument_11.svg similarity index 100% rename from tileserver/assets/sprites/monument_11.svg rename to services/tileserver/assets/sprites/monument_11.svg diff --git a/tileserver/assets/sprites/mountain_11.svg b/services/tileserver/assets/sprites/mountain_11.svg similarity index 100% rename from tileserver/assets/sprites/mountain_11.svg rename to services/tileserver/assets/sprites/mountain_11.svg diff --git a/tileserver/assets/sprites/museum_11.svg b/services/tileserver/assets/sprites/museum_11.svg similarity index 100% rename from tileserver/assets/sprites/museum_11.svg rename to services/tileserver/assets/sprites/museum_11.svg diff --git a/tileserver/assets/sprites/music_11.svg b/services/tileserver/assets/sprites/music_11.svg similarity index 100% rename from tileserver/assets/sprites/music_11.svg rename to services/tileserver/assets/sprites/music_11.svg diff --git a/tileserver/assets/sprites/oneway.svg b/services/tileserver/assets/sprites/oneway.svg similarity index 100% rename from tileserver/assets/sprites/oneway.svg rename to services/tileserver/assets/sprites/oneway.svg diff --git a/tileserver/assets/sprites/park_11.svg b/services/tileserver/assets/sprites/park_11.svg similarity index 100% rename from tileserver/assets/sprites/park_11.svg rename to services/tileserver/assets/sprites/park_11.svg diff --git a/tileserver/assets/sprites/pharmacy_11.svg b/services/tileserver/assets/sprites/pharmacy_11.svg similarity index 100% rename from tileserver/assets/sprites/pharmacy_11.svg rename to services/tileserver/assets/sprites/pharmacy_11.svg diff --git a/tileserver/assets/sprites/picnic_site_11.svg b/services/tileserver/assets/sprites/picnic_site_11.svg similarity index 100% rename from tileserver/assets/sprites/picnic_site_11.svg rename to services/tileserver/assets/sprites/picnic_site_11.svg diff --git a/tileserver/assets/sprites/pitch_11.svg b/services/tileserver/assets/sprites/pitch_11.svg similarity index 100% rename from tileserver/assets/sprites/pitch_11.svg rename to services/tileserver/assets/sprites/pitch_11.svg diff --git a/tileserver/assets/sprites/place_of_worship_11.svg b/services/tileserver/assets/sprites/place_of_worship_11.svg similarity index 100% rename from tileserver/assets/sprites/place_of_worship_11.svg rename to services/tileserver/assets/sprites/place_of_worship_11.svg diff --git a/tileserver/assets/sprites/playground_11.svg b/services/tileserver/assets/sprites/playground_11.svg similarity index 100% rename from tileserver/assets/sprites/playground_11.svg rename to services/tileserver/assets/sprites/playground_11.svg diff --git a/tileserver/assets/sprites/police_11.svg b/services/tileserver/assets/sprites/police_11.svg similarity index 100% rename from tileserver/assets/sprites/police_11.svg rename to services/tileserver/assets/sprites/police_11.svg diff --git a/tileserver/assets/sprites/post_11.svg b/services/tileserver/assets/sprites/post_11.svg similarity index 100% rename from tileserver/assets/sprites/post_11.svg rename to services/tileserver/assets/sprites/post_11.svg diff --git a/tileserver/assets/sprites/prison_11.svg b/services/tileserver/assets/sprites/prison_11.svg similarity index 100% rename from tileserver/assets/sprites/prison_11.svg rename to services/tileserver/assets/sprites/prison_11.svg diff --git a/tileserver/assets/sprites/rail_light_11.svg b/services/tileserver/assets/sprites/rail_light_11.svg similarity index 100% rename from tileserver/assets/sprites/rail_light_11.svg rename to services/tileserver/assets/sprites/rail_light_11.svg diff --git a/tileserver/assets/sprites/rail_metro_11.svg b/services/tileserver/assets/sprites/rail_metro_11.svg similarity index 100% rename from tileserver/assets/sprites/rail_metro_11.svg rename to services/tileserver/assets/sprites/rail_metro_11.svg diff --git a/tileserver/assets/sprites/railway_11.svg b/services/tileserver/assets/sprites/railway_11.svg similarity index 100% rename from tileserver/assets/sprites/railway_11.svg rename to services/tileserver/assets/sprites/railway_11.svg diff --git a/tileserver/assets/sprites/religious_christian_11.svg b/services/tileserver/assets/sprites/religious_christian_11.svg similarity index 100% rename from tileserver/assets/sprites/religious_christian_11.svg rename to services/tileserver/assets/sprites/religious_christian_11.svg diff --git a/tileserver/assets/sprites/religious_jewish_11.svg b/services/tileserver/assets/sprites/religious_jewish_11.svg similarity index 100% rename from tileserver/assets/sprites/religious_jewish_11.svg rename to services/tileserver/assets/sprites/religious_jewish_11.svg diff --git a/tileserver/assets/sprites/religious_muslim_11.svg b/services/tileserver/assets/sprites/religious_muslim_11.svg similarity index 100% rename from tileserver/assets/sprites/religious_muslim_11.svg rename to services/tileserver/assets/sprites/religious_muslim_11.svg diff --git a/tileserver/assets/sprites/restaurant_11.svg b/services/tileserver/assets/sprites/restaurant_11.svg similarity index 100% rename from tileserver/assets/sprites/restaurant_11.svg rename to services/tileserver/assets/sprites/restaurant_11.svg diff --git a/tileserver/assets/sprites/road_1.svg b/services/tileserver/assets/sprites/road_1.svg similarity index 100% rename from tileserver/assets/sprites/road_1.svg rename to services/tileserver/assets/sprites/road_1.svg diff --git a/tileserver/assets/sprites/road_2.svg b/services/tileserver/assets/sprites/road_2.svg similarity index 100% rename from tileserver/assets/sprites/road_2.svg rename to services/tileserver/assets/sprites/road_2.svg diff --git a/tileserver/assets/sprites/road_3.svg b/services/tileserver/assets/sprites/road_3.svg similarity index 100% rename from tileserver/assets/sprites/road_3.svg rename to services/tileserver/assets/sprites/road_3.svg diff --git a/tileserver/assets/sprites/road_4.svg b/services/tileserver/assets/sprites/road_4.svg similarity index 100% rename from tileserver/assets/sprites/road_4.svg rename to services/tileserver/assets/sprites/road_4.svg diff --git a/tileserver/assets/sprites/road_5.svg b/services/tileserver/assets/sprites/road_5.svg similarity index 100% rename from tileserver/assets/sprites/road_5.svg rename to services/tileserver/assets/sprites/road_5.svg diff --git a/tileserver/assets/sprites/road_6.svg b/services/tileserver/assets/sprites/road_6.svg similarity index 100% rename from tileserver/assets/sprites/road_6.svg rename to services/tileserver/assets/sprites/road_6.svg diff --git a/tileserver/assets/sprites/rocket_11.svg b/services/tileserver/assets/sprites/rocket_11.svg similarity index 100% rename from tileserver/assets/sprites/rocket_11.svg rename to services/tileserver/assets/sprites/rocket_11.svg diff --git a/tileserver/assets/sprites/school_11.svg b/services/tileserver/assets/sprites/school_11.svg similarity index 100% rename from tileserver/assets/sprites/school_11.svg rename to services/tileserver/assets/sprites/school_11.svg diff --git a/tileserver/assets/sprites/shop_11.svg b/services/tileserver/assets/sprites/shop_11.svg similarity index 100% rename from tileserver/assets/sprites/shop_11.svg rename to services/tileserver/assets/sprites/shop_11.svg diff --git a/tileserver/assets/sprites/stadium_11.svg b/services/tileserver/assets/sprites/stadium_11.svg similarity index 100% rename from tileserver/assets/sprites/stadium_11.svg rename to services/tileserver/assets/sprites/stadium_11.svg diff --git a/tileserver/assets/sprites/star_11.svg b/services/tileserver/assets/sprites/star_11.svg similarity index 100% rename from tileserver/assets/sprites/star_11.svg rename to services/tileserver/assets/sprites/star_11.svg diff --git a/tileserver/assets/sprites/suitcase_11.svg b/services/tileserver/assets/sprites/suitcase_11.svg similarity index 100% rename from tileserver/assets/sprites/suitcase_11.svg rename to services/tileserver/assets/sprites/suitcase_11.svg diff --git a/tileserver/assets/sprites/swimming_11.svg b/services/tileserver/assets/sprites/swimming_11.svg similarity index 100% rename from tileserver/assets/sprites/swimming_11.svg rename to services/tileserver/assets/sprites/swimming_11.svg diff --git a/tileserver/assets/sprites/theatre_11.svg b/services/tileserver/assets/sprites/theatre_11.svg similarity index 100% rename from tileserver/assets/sprites/theatre_11.svg rename to services/tileserver/assets/sprites/theatre_11.svg diff --git a/tileserver/assets/sprites/toilet_11.svg b/services/tileserver/assets/sprites/toilet_11.svg similarity index 100% rename from tileserver/assets/sprites/toilet_11.svg rename to services/tileserver/assets/sprites/toilet_11.svg diff --git a/tileserver/assets/sprites/town_hall_11.svg b/services/tileserver/assets/sprites/town_hall_11.svg similarity index 100% rename from tileserver/assets/sprites/town_hall_11.svg rename to services/tileserver/assets/sprites/town_hall_11.svg diff --git a/tileserver/assets/sprites/triangle_11.svg b/services/tileserver/assets/sprites/triangle_11.svg similarity index 100% rename from tileserver/assets/sprites/triangle_11.svg rename to services/tileserver/assets/sprites/triangle_11.svg diff --git a/tileserver/assets/sprites/triangle_stroked_11.svg b/services/tileserver/assets/sprites/triangle_stroked_11.svg similarity index 100% rename from tileserver/assets/sprites/triangle_stroked_11.svg rename to services/tileserver/assets/sprites/triangle_stroked_11.svg diff --git a/tileserver/assets/sprites/us-highway_1.svg b/services/tileserver/assets/sprites/us-highway_1.svg similarity index 100% rename from tileserver/assets/sprites/us-highway_1.svg rename to services/tileserver/assets/sprites/us-highway_1.svg diff --git a/tileserver/assets/sprites/us-highway_2.svg b/services/tileserver/assets/sprites/us-highway_2.svg similarity index 100% rename from tileserver/assets/sprites/us-highway_2.svg rename to services/tileserver/assets/sprites/us-highway_2.svg diff --git a/tileserver/assets/sprites/us-highway_3.svg b/services/tileserver/assets/sprites/us-highway_3.svg similarity index 100% rename from tileserver/assets/sprites/us-highway_3.svg rename to services/tileserver/assets/sprites/us-highway_3.svg diff --git a/tileserver/assets/sprites/us-interstate_1.svg b/services/tileserver/assets/sprites/us-interstate_1.svg similarity index 100% rename from tileserver/assets/sprites/us-interstate_1.svg rename to services/tileserver/assets/sprites/us-interstate_1.svg diff --git a/tileserver/assets/sprites/us-interstate_2.svg b/services/tileserver/assets/sprites/us-interstate_2.svg similarity index 100% rename from tileserver/assets/sprites/us-interstate_2.svg rename to services/tileserver/assets/sprites/us-interstate_2.svg diff --git a/tileserver/assets/sprites/us-interstate_3.svg b/services/tileserver/assets/sprites/us-interstate_3.svg similarity index 100% rename from tileserver/assets/sprites/us-interstate_3.svg rename to services/tileserver/assets/sprites/us-interstate_3.svg diff --git a/tileserver/assets/sprites/us-state_1.svg b/services/tileserver/assets/sprites/us-state_1.svg similarity index 100% rename from tileserver/assets/sprites/us-state_1.svg rename to services/tileserver/assets/sprites/us-state_1.svg diff --git a/tileserver/assets/sprites/us-state_2.svg b/services/tileserver/assets/sprites/us-state_2.svg similarity index 100% rename from tileserver/assets/sprites/us-state_2.svg rename to services/tileserver/assets/sprites/us-state_2.svg diff --git a/tileserver/assets/sprites/us-state_3.svg b/services/tileserver/assets/sprites/us-state_3.svg similarity index 100% rename from tileserver/assets/sprites/us-state_3.svg rename to services/tileserver/assets/sprites/us-state_3.svg diff --git a/tileserver/assets/sprites/us-state_4.svg b/services/tileserver/assets/sprites/us-state_4.svg similarity index 100% rename from tileserver/assets/sprites/us-state_4.svg rename to services/tileserver/assets/sprites/us-state_4.svg diff --git a/tileserver/assets/sprites/us-state_5.svg b/services/tileserver/assets/sprites/us-state_5.svg similarity index 100% rename from tileserver/assets/sprites/us-state_5.svg rename to services/tileserver/assets/sprites/us-state_5.svg diff --git a/tileserver/assets/sprites/us-state_6.svg b/services/tileserver/assets/sprites/us-state_6.svg similarity index 100% rename from tileserver/assets/sprites/us-state_6.svg rename to services/tileserver/assets/sprites/us-state_6.svg diff --git a/tileserver/assets/sprites/veterinary_11.svg b/services/tileserver/assets/sprites/veterinary_11.svg similarity index 100% rename from tileserver/assets/sprites/veterinary_11.svg rename to services/tileserver/assets/sprites/veterinary_11.svg diff --git a/tileserver/assets/sprites/volcano_11.svg b/services/tileserver/assets/sprites/volcano_11.svg similarity index 100% rename from tileserver/assets/sprites/volcano_11.svg rename to services/tileserver/assets/sprites/volcano_11.svg diff --git a/tileserver/assets/sprites/wave.svg b/services/tileserver/assets/sprites/wave.svg similarity index 100% rename from tileserver/assets/sprites/wave.svg rename to services/tileserver/assets/sprites/wave.svg diff --git a/tileserver/assets/sprites/zoo_11.svg b/services/tileserver/assets/sprites/zoo_11.svg similarity index 100% rename from tileserver/assets/sprites/zoo_11.svg rename to services/tileserver/assets/sprites/zoo_11.svg diff --git a/tileserver/image/config.json.template b/services/tileserver/config.json.template similarity index 86% rename from tileserver/image/config.json.template rename to services/tileserver/config.json.template index 5e9286668..af15699ff 100644 --- a/tileserver/image/config.json.template +++ b/services/tileserver/config.json.template @@ -12,8 +12,8 @@ }, "styles": { - "bright": { - "style": "/app/styles/bright.json", + "style": { + "style": "/app/styles/style.json", "serve_data": true } }, diff --git a/tileserver/image/configure_run.sh b/services/tileserver/configure_run.sh similarity index 67% rename from tileserver/image/configure_run.sh rename to services/tileserver/configure_run.sh index d431b5737..e11524015 100755 --- a/tileserver/image/configure_run.sh +++ b/services/tileserver/configure_run.sh @@ -3,7 +3,7 @@ set -xe envsubst < /app/config.json.template > /app/config.json -envsubst < /data/styles/bright.json.template > /app/styles/bright.json +envsubst < /styles/style.json.template > /app/styles/style.json cat /app/config.json diff --git a/services/tileserver/init.sh b/services/tileserver/init.sh new file mode 100755 index 000000000..dab77cc77 --- /dev/null +++ b/services/tileserver/init.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -xe + +sleep 5 + +mkdir -p $(dirname ${FONT_ARTIFACT_DEST_PATH}) +mkdir -p $(dirname ${SPRITE_ARTIFACT_DEST_PATH}) +mkdir -p $(dirname ${MBTILES_ARTIFACT_DEST_PATH}) + +if [ -f "${FONT_ARTIFACT_DEST_PATH}" ]; then + echo "Nothing to do, already have ${FONT_ARTIFACT_DEST_PATH}" +elif [ -f "${FONT_ARTIFACT_SOURCE_PATH}" ]; then + echo "Copying font artifact." + cp "${FONT_ARTIFACT_SOURCE_PATH}" "${FONT_ARTIFACT_DEST_PATH}" +else + echo "Downloading font artifact." + wget -O "${FONT_ARTIFACT_DEST_PATH}" "${FONT_ARTIFACT_URL}" +fi + +cd $(dirname ${FONT_ARTIFACT_DEST_PATH}) && tar xvf ${FONT_ARTIFACT_DEST_PATH} + +if [ -f "${SPRITE_ARTIFACT_DEST_PATH}" ]; then + echo "Nothing to do, already have ${SPRITE_ARTIFACT_DEST_PATH}" +elif [ -f "${SPRITE_ARTIFACT_SOURCE_PATH}" ]; then + echo "Copying sprite artifact." + cp "${SPRITE_ARTIFACT_SOURCE_PATH}" "${SPRITE_ARTIFACT_DEST_PATH}" +else + echo "Downloading sprite artifact." + wget -O "${SPRITE_ARTIFACT_DEST_PATH}" "${SPRITE_ARTIFACT_URL}" +fi + +cd $(dirname ${SPRITE_ARTIFACT_DEST_PATH}) && tar xvf ${SPRITE_ARTIFACT_DEST_PATH} + +if [ -f "${MBTILES_ARTIFACT_DEST_PATH}" ]; then + echo "Nothing to do, already have ${MBTILES_ARTIFACT_DEST_PATH}" +elif [ -f "${MBTILES_ARTIFACT_SOURCE_PATH}" ]; then + echo "Copying sprite artifact." + cp "${MBTILES_ARTIFACT_SOURCE_PATH}" "${MBTILES_ARTIFACT_DEST_PATH}" +else + echo "Downloading sprite artifact." + wget -O "${MBTILES_ARTIFACT_DEST_PATH}" "${MBTILES_ARTIFACT_URL}" +fi diff --git a/tileserver/style/LICENSE.md b/services/tileserver/style/LICENSE.md similarity index 100% rename from tileserver/style/LICENSE.md rename to services/tileserver/style/LICENSE.md diff --git a/tileserver/style/style.json.template b/services/tileserver/style/style.json.template similarity index 100% rename from tileserver/style/style.json.template rename to services/tileserver/style/style.json.template diff --git a/services/valhalla/init.sh b/services/valhalla/init.sh new file mode 100755 index 000000000..22ccd6c35 --- /dev/null +++ b/services/valhalla/init.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -xe + + +if [ -f "${ARTIFACT_LOCK}" ]; then + echo "Nothing to do, already have artifact." + exit 0 +fi + +mkdir -p /data/valhalla/ + +if [ -f "${ARTIFACT_SOURCE_PATH}" ]; then + echo "Copying artifact." + cd /data && cat ${ARTIFACT_SOURCE_PATH} | pbzip2 -d > /data/valhalla/tiles.tar +else + echo "Downloading artifact." + cd /data && wget -qO- "${ARTIFACT_URL}" | pbzip2 -d > /data/valhalla/tiles.tar +fi + +valhalla_build_config --mjolnir-timezone /data/timezones.sqlite --mjolnir-admin /data/admins.sqlite > /data/valhalla.json + +chown -R valhalla /data + +touch ${ARTIFACT_LOCK} + +ls -lR /data diff --git a/tileserver/image/Dockerfile b/tileserver/image/Dockerfile deleted file mode 100644 index 17248fc41..000000000 --- a/tileserver/image/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM node:16 - -RUN npm install -g tileserver-gl-light - -USER root - -RUN apt-get update -y && apt-get install -y gettext-base - -RUN mkdir -p /app/styles -RUN chown node /app -RUN chown node /app/styles -USER node - -COPY configure_run.sh config.json.template /app/ - -ENTRYPOINT [ "/bin/bash" ] - -CMD [ "/app/configure_run.sh" ] diff --git a/valhalla/build/.gitignore b/valhalla/build/.gitignore deleted file mode 100644 index 42e76ae86..000000000 --- a/valhalla/build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -data.osm.pbf \ No newline at end of file diff --git a/valhalla/build/Dockerfile b/valhalla/build/Dockerfile deleted file mode 100644 index 60e722541..000000000 --- a/valhalla/build/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM gisops/valhalla:latest - -USER root -RUN apt-get -y update && apt-get install -y pbzip2 -WORKDIR /tiles -RUN chown valhalla /tiles -USER valhalla - -RUN valhalla_build_config --mjolnir-tile-dir /tiles --mjolnir-timezone /tiles/timezones.sqlite --mjolnir-admin /tiles/admins.sqlite > valhalla.json -RUN valhalla_build_timezones > /tiles/timezones.sqlite - -COPY data.osm.pbf /tiles/ - -RUN valhalla_build_tiles -c valhalla.json /tiles/data.osm.pbf - -RUN bash -c 'cd /tiles && ls | tar -c --files-from - | pbzip2 -c -6 > /tiles/valhalla.tar.bz2' diff --git a/valhalla/run/Dockerfile b/valhalla/run/Dockerfile deleted file mode 100644 index bc1a9b6c1..000000000 --- a/valhalla/run/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM gisops/valhalla:latest - -USER root -WORKDIR /data/valhalla -RUN chown valhalla /data -RUN chown valhalla /data/valhalla -USER valhalla - -COPY gen_hash.sh /app/gen_hash.sh - -ENTRYPOINT ["/bin/bash"] - -USER valhalla - -CMD ["/app/gen_hash.sh"] diff --git a/valhalla/run/gen_hash.sh b/valhalla/run/gen_hash.sh deleted file mode 100755 index 92bec903f..000000000 --- a/valhalla/run/gen_hash.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -xe - -mkdir -p /data/valhalla/ - -cd /valhalla_data && time cat /data_mount/${HEADWAY_AREA}.valhalla.tar.bz2 | pbzip2 -d | tar x - -valhalla_build_config --mjolnir-timezone /valhalla_data/timezones.sqlite --mjolnir-admin /valhalla_data/admins.sqlite > valhalla.json - -valhalla_service valhalla.json diff --git a/web/frontend/.dockerignore b/web/frontend/.earthlyignore similarity index 100% rename from web/frontend/.dockerignore rename to web/frontend/.earthlyignore diff --git a/web/frontend/src/components/BaseMap.vue b/web/frontend/src/components/BaseMap.vue index cb81a8089..c8b4ccfa2 100644 --- a/web/frontend/src/components/BaseMap.vue +++ b/web/frontend/src/components/BaseMap.vue @@ -42,7 +42,7 @@ async function loadMap() { ]; map = new maplibregl.Map({ container: 'map', // container id - style: '/styles/bright/style.json', // style URL + style: '/styles/style/style.json', // style URL center: [0, 0], // starting position [lng, lat] maxBounds: maxBounds, zoom: 1, // starting zoom From 500ef7ed40e129a9993c268ca9d9b12861e81a56 Mon Sep 17 00:00:00 2001 From: Ellen Poe Date: Thu, 11 Aug 2022 19:20:02 -0700 Subject: [PATCH 2/5] Add a --no-updates flag to nominatim import to reduce disk usage --- Earthfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index c373613a4..ba94176df 100644 --- a/Earthfile +++ b/Earthfile @@ -71,7 +71,9 @@ nominatim-build: RUN bash -c 'useradd -m -p ${NOMINATIM_PASSWORD} nominatim' ENV PBF_PATH=/data.osm.pbf RUN bash -c '/app/config.sh' - RUN bash -c '/app/init.sh' + # This is probably the worst line of code in the whole project. + # Piggback off the THREADS param to also pass in a flag that drops update tables. + RUN THREADS="$(nproc) --no-updates" bash -c '/app/init.sh' RUN touch /var/lib/postgresql/12/main/import-finished From e99c77f2c96baf3538802250159bfa6e668ba457 Mon Sep 17 00:00:00 2001 From: Ellen Poe Date: Sat, 13 Aug 2022 14:57:54 -0700 Subject: [PATCH 3/5] Update docs and readme --- BUILD.md | 34 ++++++++++++++++++++++++++-------- Earthfile | 2 +- README.md | 2 +- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/BUILD.md b/BUILD.md index ccf476e0a..e7636bf72 100644 --- a/BUILD.md +++ b/BUILD.md @@ -1,20 +1,38 @@ +# Building Headway + +Setting up your own Headway instance should be fairly straightforward if you follow these docs. Feel free to open bugs if things go wrong, or PRs to improve the project though! + +## Install Earthly + +Headway processes data and builds its Docker images using Earthly. Visit [earthly.dev](https://earthly.dev/) for information about Earthly, and documentation on how to install it. + ## Supported Build Methods -Headway can be built using a BBBike extract if one exists for a metro area you're interested in, or you can supply your own `.osm.pbf` file to cover larger areas like US states or European countries. +Headway can be built using a BBBike extract if one exists for a metro area you're interested in, or you can supply your own `.osm.pbf` file to cover areas that BBBike doesn't cover, or larger areas like US states or European countries. ### Building Headway from a supported BBBike extract -1. Pick a metro area from the `Makefile` like "Amsterdam" or "Denver". These values are case-sensitive. +This section pertains to builds from BBBike extracts. Skip this if you know you need to bring your own OpenStreetMap extract. + +#### Currently supported cities + +Headway currently supports fully automatic builds for the following cities: + +Aachen, Aarhus, Adelaide, Albuquerque, Alexandria, Amsterdam, Antwerpen, Arnhem, Auckland, Augsburg, Austin, Baghdad, Baku, Balaton, Bamberg, Bangkok, Barcelona, Basel, Beijing, Beirut, Berkeley, Berlin, Bern, Bielefeld, Birmingham, Bochum, Bogota, Bombay, Bonn, Bordeaux, Boulder, BrandenburgHavel, Braunschweig, Bremen, Bremerhaven, Brisbane, Bristol, Brno, Bruegge, Bruessel, Budapest, BuenosAires, Cairo, Calgary, Cambridge, CambridgeMa, Canberra, CapeTown, Chemnitz, Chicago, ClermontFerrand, Colmar, Copenhagen, Cork, Corsica, Corvallis, Cottbus, Cracow, CraterLake, Curitiba, Cusco, Dallas, Darmstadt, Davis, DenHaag, Denver, Dessau, Dortmund, Dresden, Dublin, Duesseldorf, Duisburg, Edinburgh, Eindhoven, Emden, Erfurt, Erlangen, Eugene, Flensburg, FortCollins, Frankfurt, FrankfurtOder, Freiburg, Gdansk, Genf, Gent, Gera, Glasgow, Gliwice, Goerlitz, Goeteborg, Goettingen, Graz, Groningen, Halifax, Halle, Hamburg, Hamm, Hannover, Heilbronn, Helsinki, Hertogenbosch, Huntsville, Innsbruck, Istanbul, Jena, Jerusalem, Johannesburg, Kaiserslautern, Karlsruhe, Kassel, Katowice, Kaunas, Kiel, Kiew, Koblenz, Koeln, Konstanz, LakeGarda, LaPaz, LaPlata, Lausanne, Leeds, Leipzig, Lima, Linz, Lisbon, Liverpool, Ljubljana, Lodz, London, Luebeck, Luxemburg, Lyon, Maastricht, Madison, Madrid, Magdeburg, Mainz, Malmoe, Manchester, Mannheim, Marseille, Melbourne, Memphis, MexicoCity, Miami, Minsk, Moenchengladbach, Montevideo, Montpellier, Montreal, Moscow, Muenchen, Muenster, NewDelhi, NewOrleans, NewYork, Nuernberg, Oldenburg, Oranienburg, Orlando, Oslo, Osnabrueck, Ostrava, Ottawa, Paderborn, Palma, PaloAlto, Paris, Perth, Philadelphia, PhnomPenh, Portland, PortlandME, Porto, PortoAlegre, Potsdam, Poznan, Prag, Providence, Regensburg, Riga, RiodeJaneiro, Rostock, Rotterdam, Ruegen, Saarbruecken, Sacramento, Saigon, Salzburg, SanFrancisco, SanJose, SanktPetersburg, SantaBarbara, SantaCruz, Santiago, Sarajewo, Schwerin, Seattle, Seoul, Sheffield, Singapore, Sofia, Stockholm, Stockton, Strassburg, Stuttgart, Sucre, Sydney, Szczecin, Tallinn, Tehran, Tilburg, Tokyo, Toronto, Toulouse, Trondheim, Tucson, Turin, UlanBator, Ulm, Usedom, Utrecht, Vancouver, Victoria, WarenMueritz, Warsaw, WashingtonDC, Waterloo, Wien, Wroclaw, Wuerzburg, Wuppertal, Zagreb, Zuerich + +#### Build procedure. + +1. Pick a metro area from the list above, like "Amsterdam" or "Denver". These values are case-sensitive. 2. (Optional) Set up GTFS feeds for trip planning and multimodal routing capability. This dramatically increases hardware requirements for large metro areas. - 1. Run `make Amsterdam.enumerate_gtfs_feeds`, replacing "Amsterdam" with your metro area of choice. + 1. Run `earthly +gtfs-enumerate --area="Amsterdam"`, replacing "Amsterdam" with your metro area of choice. 2. Examine `data/Amsterdam.gtfs.csv` and manually edit it if necessary to curate GTFS feeds. Some may have errors, and many may be useless for your purposes. -3. Execute `make Amsterdam` using your chosen metro area. +3. Execute `earthly +build --area="Amsterdam"` using your chosen metro area. 4. Make a `.env` file with configuration. See/copy `.env.example` for defaults. In particular: 1. `HEADWAY_PUBLIC_URL` (Optional for fully local setup that is accessed exclusively via the base URL `http://127.0.0.1`) Pick a base URL for the domain you wish to serve on, paying attention to scheme (http vs https), domain and port (if not default). This will look like "https://example.com", "http://maps.my.cool.intranet" or "https://maps.example.com:8080". Please omit the trailing slash. - 2. `HEADWAY_AREA`: The name of the area you ran above. + 2. `HEADWAY_AREA`: The name of the area you ran above. In our case, "Amsterdam". 5. Execute `docker-compose up -d` to bring up a headway server on port 8080. 6. (For https and non-default port use only) reverse-proxy traffic to port 8080. @@ -23,10 +41,10 @@ That's it! In the future I'd like to have a kubernetes config to further product ### Building Headway from your own OSM extract -Using a custom OSM extract is a bit more complicated, and less regularly tested. Please report issues if you have any, though. Transit trip planning isn't currently supported for arbitrary OSM extracts, contributions are welcome though! +Using a custom OSM extract is a bit more complicated, and less regularly tested. Please report issues if you have any. Transit trip planning isn't currently supported for arbitrary OSM extracts, contributions are welcome though! -1. Copy your OSM extract into the `data/` directory, as e.g. `data/california.osm.pbf`. -2. Execute `make california.custom` replacing `california` with the name (no extension) of your OSM extract. +1. Copy your OSM extract into Headway's top-level directory (same directory as this file), as e.g. `./california.osm.pbf`. It is important to name it something different than the cities listed above. For example, if I was building a custom extract of Amsterdam to avoid conflicts I would name it `AmsterdamCustom`. +2. Execute `earthly +build --area="california"` replacing `california` with the name (no extension) of your OSM extract. 3. Make a `.env` file with configuration. See/copy `.env.example` for defaults. In particular: 1. `HEADWAY_PUBLIC_URL` (Optional for fully local setup that is accessed exclusively via the base URL `http://127.0.0.1`) Pick a base URL for the domain you wish to serve on, paying attention to scheme (http vs https), domain and port (if not default). This will look like "https://example.com", "http://maps.my.cool.intranet" or "https://maps.example.com:8080". Please omit the trailing slash. diff --git a/Earthfile b/Earthfile index ba94176df..edf3581d7 100644 --- a/Earthfile +++ b/Earthfile @@ -5,7 +5,7 @@ VERSION 0.6 # OSM extract ############################## -metro: +build: FROM debian:bullseye-slim ARG area COPY (+extract/data.osm.pbf --area=${area}) /data.osm.pbf diff --git a/README.md b/README.md index ffcad4ee1..473e992e7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/ellenhp/headway) ![GitHub](https://img.shields.io/github/license/ellenhp/headway) -Headway is a maps stack in a box that makes it easy to take your location data into your own hands. For example, `make Amsterdam` then `docker-compose up` will bring up a fully functional maps web app for the Amsterdam metro area. This includes a frontend, basemap, geocoder and routing engine. Over 200 different cities are currently supported. +Headway is a maps stack in a box that makes it easy to take your location data into your own hands. With just a few commands you can bring up your own fully functional maps server. This includes a frontend, basemap, geocoder and routing engine. Over 200 different cities are currently supported. See [BUILD.md](./BUILD.md) for more information about the build process. From d366ae0fe02dfc589aeaca845742d739543e58ce Mon Sep 17 00:00:00 2001 From: Ellen Poe Date: Sat, 13 Aug 2022 15:41:14 -0700 Subject: [PATCH 4/5] Fix transit with Earthly builds --- Earthfile | 4 +++- services/gtfs/download_gtfs_feeds.py | 4 +++- services/otp/run_otp.sh | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Earthfile b/Earthfile index edf3581d7..32a2f791c 100644 --- a/Earthfile +++ b/Earthfile @@ -236,7 +236,7 @@ gtfs-build: FROM +gtfs-base COPY ./services/gtfs/download_gtfs_feeds.py /gtfs/ ARG area - COPY --if-exists build/${area}.gtfs_feeds.csv /gtfs/gtfs_feeds.csv + COPY --if-exists data/${area}.gtfs_feeds.csv /gtfs/gtfs_feeds.csv RUN touch /gtfs/gtfs_feeds.csv # Just in case the GTFS feeds weren't enumerated earlier. RUN python /gtfs/download_gtfs_feeds.py RUN bash -c "cd /gtfs_feeds && ls *.zip | tar -cf /gtfs/gtfs.tar --files-from -" @@ -291,6 +291,8 @@ otp-init-image: otp-serve-image: FROM +otp-base + RUN apt-get update -y && apt-get install -y --no-install-recommends netcat + ARG javaMemArgs=-Xmx4G COPY ./services/otp/run_otp.sh /otp diff --git a/services/gtfs/download_gtfs_feeds.py b/services/gtfs/download_gtfs_feeds.py index 7da9133a9..81ea18667 100644 --- a/services/gtfs/download_gtfs_feeds.py +++ b/services/gtfs/download_gtfs_feeds.py @@ -7,5 +7,7 @@ reader = csv.reader(f.readlines(), delimiter=',', quotechar='"') for line in reader: print("Downloading feed for", line[0].strip()) - with open('/gtfs_feeds/' + line[1].strip() + '.gtfs.zip', 'wb') as f: + filename = '/gtfs_feeds/' + line[1].strip() + '.gtfs.zip' + with open(filename, 'wb') as f: + print("Writing feed to", filename) f.write(requests.get(line[2].strip()).content) diff --git a/services/otp/run_otp.sh b/services/otp/run_otp.sh index 969c8b511..e2588f257 100755 --- a/services/otp/run_otp.sh +++ b/services/otp/run_otp.sh @@ -5,9 +5,10 @@ set -xe if [[ -s /data/${HEADWAY_AREA}.graph.obj ]]; then netcat -l -p 9999 # Hack: Signal to clients that we have a transit graph. - echo "{\"transitServiceStart\": \"-P10Y\", \"streetGraph\": \"/data/${HEADWAY_AREA}.graph.obj\"}" > /otp/build-config.json + mkdir /servedir + ln -s /data/${HEADWAY_AREA}.graph.obj /servedir/graph.obj - java ${JAVA_MEM_ARGS} -jar otp-shaded.jar --load /otp + java ${JAVA_MEM_ARGS} -jar /otp/otp-shaded.jar --load /servedir else while : do From 4b71973bc7e1df280a480663eb83f0b4d70a8c59 Mon Sep 17 00:00:00 2001 From: Ellen Poe Date: Sat, 13 Aug 2022 16:11:05 -0700 Subject: [PATCH 5/5] Update BUILD.md with caveat for docker-compose restarts --- BUILD.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/BUILD.md b/BUILD.md index e7636bf72..bbce5988a 100644 --- a/BUILD.md +++ b/BUILD.md @@ -53,3 +53,9 @@ Using a custom OSM extract is a bit more complicated, and less regularly tested. 4. Execute `docker-compose up -d` to bring up a headway server on port 8080. 5. (For https and non-default port use only) reverse-proxy traffic to port 8080. + +## Docker-compose restarts + +Because Headway's docker-compose configuration uses init containers to populate a docker volume containing internal data, rebuilding the data for a metro area won't update existing containers. It's necessary to run `docker-compose down --volume` to re-initialize the data in the init containers. + +This is necessary whenever you rebuild the data for a metro area, or change which area you're serving data for in the `.env` file.