diff --git a/images/backup-restore/Dockerfile b/images/backup-restore/Dockerfile index 95996a98..7179b68b 100644 --- a/images/backup-restore/Dockerfile +++ b/images/backup-restore/Dockerfile @@ -1,13 +1,13 @@ FROM python:3.9.9-slim -# Install Postgres client, GCP CLI, and Azure CLI +# Install Postgres client 17, GCP CLI, Azure CLI, and ZSTD compression tools RUN apt-get update \ && apt-get install -y curl apt-transport-https lsb-release gnupg \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ && apt-get update \ - && apt-get install -y postgresql-client-14 wget \ + && apt-get install -y postgresql-client-17 wget zstd \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/images/db/Dockerfile b/images/db/Dockerfile index a7e6a566..ed35e853 100644 --- a/images/db/Dockerfile +++ b/images/db/Dockerfile @@ -1,18 +1,39 @@ -FROM postgres:14 -RUN apt-get update \ - && apt-get install -y \ - postgresql-server-dev-14 \ - make \ +# Stage 1: Compilar el plugin para PostgreSQL 17 +FROM postgres:17 AS builder + +# Instalar dependencias de compilación +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ build-essential \ - postgresql-14-postgis-3 \ - && apt-get clean && rm -rf /var/lib/apt/lists/* + cmake \ + git \ + libboost-program-options-dev \ + libbz2-dev \ + libexpat1-dev \ + libosmium2-dev \ + libprotozero-dev \ + libyaml-cpp-dev \ + libpqxx-dev \ + postgresql-server-dev-17 \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Clonar y compilar el plugin osmdbt para PostgreSQL 17 +RUN git clone https://github.com/openstreetmap/osmdbt.git /tmp/osmdbt && \ + cd /tmp/osmdbt && \ + git checkout v0.9 && \ + cd postgresql-plugin && \ + mkdir -p build && cd build && \ + cmake .. && \ + make + +FROM postgres:17 + +COPY --from=builder /tmp/osmdbt/postgresql-plugin/build/osm-logical.so /usr/lib/postgresql/17/lib/osm-logical.so + +RUN ln -s /usr/lib/postgresql/17/lib/osm-logical.so /usr/lib/postgresql/17/lib/osm_logical.so -ADD functions/functions.sql /usr/local/share/osm-db-functions.sql -ADD docker_postgres.sh /docker-entrypoint-initdb.d/ -RUN mkdir -p db -RUN mkdir -p lib -ADD functions/ db/functions/ -ADD lib/quad_tile/ lib/quad_tile/ -RUN make -C db/functions/ -RUN chown -R postgres lib/ -RUN chown -R postgres db/ \ No newline at end of file +RUN chown postgres:postgres /usr/lib/postgresql/17/lib/osm-logical.so && \ + chown postgres:postgres /usr/lib/postgresql/17/lib/osm_logical.so && \ + chmod 755 /usr/lib/postgresql/17/lib/osm-logical.so && \ + chmod 755 /usr/lib/postgresql/17/lib/osm_logical.so diff --git a/images/db/README.md b/images/db/README.md index 75fb3091..9420bb9a 100644 --- a/images/db/README.md +++ b/images/db/README.md @@ -1,6 +1,8 @@ # Docker setup for postgres database instance -The OSM `API server` database - builds off a postgres 10 docker image, and installs custom functions needed by `openstreetmap`. +The OSM `API server` database - builds off a **PostgreSQL 17** docker image, and installs custom functions needed by `openstreetmap`. + +The database includes the **osmdbt plugin v0.9** for logical replication support. The functions currently are copied over from the `openstreetmap-website` code-base. There should ideally be a better way to do this. @@ -35,3 +37,8 @@ In order to run this container we need environment variables, these can be found ```sh pg_isready -h 127.0.0.1 -p 5432 ``` + +### Installed Components + +- **PostgreSQL**: 17 +- **osmdbt plugin**: v0.9 (osm_logical replication plugin) diff --git a/images/db/docker_postgres.sh b/images/db/docker_postgres.sh deleted file mode 100644 index 33ab2480..00000000 --- a/images/db/docker_postgres.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e - -# From: https://github.com/openstreetmap/openstreetmap-website/blob/af273f5d6ae160de0001ce1ac0c087d92a2463c6/docker/postgres/openstreetmap-postgres-init.sh -# Create 'openstreetmap' user -# Password and superuser privilege are needed to successfully run test suite -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" <<-EOSQL - CREATE USER openstreetmap SUPERUSER PASSWORD 'openstreetmap'; - GRANT ALL PRIVILEGES ON DATABASE $POSTGRES_DB TO openstreetmap; -EOSQL - -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE ROLE hosm WITH SUPERUSER" $POSTGRES_DB -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE EXTENSION btree_gist" $POSTGRES_DB -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE EXTENSION postgis" $POSTGRES_DB -make -C /db/functions libpgosm.so -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" $POSTGRES_DB -# psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT" openstreetmap -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" $POSTGRES_DB - - -# Define custom functions -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -f "/usr/local/share/osm-db-functions.sql" $POSTGRES_DB diff --git a/images/db/functions/Makefile b/images/db/functions/Makefile deleted file mode 100644 index dac1889d..00000000 --- a/images/db/functions/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -PG_CONFIG ?= pg_config -DESTDIR ?= . - -QTDIR=../../lib/quad_tile - -OS=$(shell uname -s) -ifeq (${OS},Darwin) - LDFLAGS=-bundle -else - LDFLAGS=-shared -endif - -all: ${DESTDIR}/libpgosm.so - -clean: - $(RM) ${DESTDIR}/*.so ${DESTDIR}/*.o - -${DESTDIR}/libpgosm.so: ${DESTDIR}/quadtile.o ${DESTDIR}/maptile.o ${DESTDIR}/xid_to_int4.o - cc ${LDFLAGS} -o $@ $^ - -${DESTDIR}/%.o: %.c - cc -I `${PG_CONFIG} --includedir` -I `${PG_CONFIG} --includedir-server` -I${QTDIR} -fPIC -O3 -DUSE_PGSQL -c -o $@ $< - -${DESTDIR}/quadtile.o: ${QTDIR}/quad_tile.h diff --git a/images/db/functions/functions.sql b/images/db/functions/functions.sql deleted file mode 100644 index 2afc0960..00000000 --- a/images/db/functions/functions.sql +++ /dev/null @@ -1,43 +0,0 @@ --------------------------------------------------------------------------------- --- From https://github.com/openstreetmap/openstreetmap-website/blob/af273f5d6ae160de0001ce1ac0c087d92a2463c6/db/functions/functions.sql --- SQL versions of the C database functions. --- --- Pure pl/pgsql versions are *slower* than the C versions, and not recommended --- for production use. However, they are significantly easier to install, and --- require fewer dependencies. --------------------------------------------------------------------------------- - --- tile_for_point function returns a Morton-encoded integer representing a z16 --- tile which contains the given (scaled_lon, scaled_lat) coordinate. Note that --- these are passed into the function as (lat, lon) and should be scaled by --- 10^7. --- --- The Morton encoding packs two dimensions down to one with fairly good --- spatial locality, and can be used to index points without the need for a --- proper 2D index. -CREATE OR REPLACE FUNCTION tile_for_point(scaled_lat int4, scaled_lon int4) - RETURNS int8 - AS $$ -DECLARE - x int8; -- quantized x from lon, - y int8; -- quantized y from lat, -BEGIN - x := round(((scaled_lon / 10000000.0) + 180.0) * 65535.0 / 360.0); - y := round(((scaled_lat / 10000000.0) + 90.0) * 65535.0 / 180.0); - - -- these bit-masks are special numbers used in the bit interleaving algorithm. - -- see https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN - -- for the original algorithm and more details. - x := (x | (x << 8)) & 16711935; -- 0x00FF00FF - x := (x | (x << 4)) & 252645135; -- 0x0F0F0F0F - x := (x | (x << 2)) & 858993459; -- 0x33333333 - x := (x | (x << 1)) & 1431655765; -- 0x55555555 - - y := (y | (y << 8)) & 16711935; -- 0x00FF00FF - y := (y | (y << 4)) & 252645135; -- 0x0F0F0F0F - y := (y | (y << 2)) & 858993459; -- 0x33333333 - y := (y | (y << 1)) & 1431655765; -- 0x55555555 - - RETURN (x << 1) | y; -END; -$$ LANGUAGE plpgsql IMMUTABLE; diff --git a/images/db/functions/maptile.c b/images/db/functions/maptile.c deleted file mode 100644 index 358a3ba7..00000000 --- a/images/db/functions/maptile.c +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include -#include - -Datum -maptile_for_point(PG_FUNCTION_ARGS) -{ - double lat = PG_GETARG_INT64(0) / 10000000.0; - double lon = PG_GETARG_INT64(1) / 10000000.0; - int zoom = PG_GETARG_INT32(2); - double scale = pow(2, zoom); - double r_per_d = M_PI / 180; - unsigned int x; - unsigned int y; - - x = floor((lon + 180.0) * scale / 360.0); - y = floor((1 - log(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / M_PI) * scale / 2.0); - - PG_RETURN_INT32((x << zoom) | y); -} - -PG_FUNCTION_INFO_V1(maptile_for_point); - -/* - * To bind this into PGSQL, try something like: - * - * CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 - * AS '/path/to/rails-port/db/functions/libpgosm', 'maptile_for_point' - * LANGUAGE C STRICT; - * - * (without all the *s) - */ - -#ifdef PG_MODULE_MAGIC -PG_MODULE_MAGIC; -#endif diff --git a/images/db/functions/quadtile.c b/images/db/functions/quadtile.c deleted file mode 100644 index 748e0b9a..00000000 --- a/images/db/functions/quadtile.c +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include -#include -#include - -Datum -tile_for_point(PG_FUNCTION_ARGS) -{ - double lat = PG_GETARG_INT32(0) / 10000000.0; - double lon = PG_GETARG_INT32(1) / 10000000.0; - - PG_RETURN_INT64(xy2tile(lon2x(lon), lat2y(lat))); -} - -PG_FUNCTION_INFO_V1(tile_for_point); - -/* - * To bind this into PGSQL, try something like: - * - * CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 - * AS '/path/to/rails-port/db/functions/libpgosm', 'tile_for_point' - * LANGUAGE C STRICT; - * - * (without all the *s) - */ diff --git a/images/db/functions/xid_to_int4.c b/images/db/functions/xid_to_int4.c deleted file mode 100644 index 9f277773..00000000 --- a/images/db/functions/xid_to_int4.c +++ /dev/null @@ -1,23 +0,0 @@ - -#include -#include - -Datum -xid_to_int4(PG_FUNCTION_ARGS) -{ - TransactionId xid = PG_GETARG_INT32(0); - - PG_RETURN_INT32(xid); -} - -PG_FUNCTION_INFO_V1(xid_to_int4); - -/* - * To bind this into PGSQL, try something like: - * - * CREATE FUNCTION xid_to_int4(xid) RETURNS int4 - * AS '/path/to/rails-port/db/functions/libpgosm', 'xid_to_int4' - * LANGUAGE C IMMUTABLE STRICT; - * - * (without all the *s) - */ diff --git a/images/db/lib/quad_tile/.gitignore b/images/db/lib/quad_tile/.gitignore deleted file mode 100644 index 978f071e..00000000 --- a/images/db/lib/quad_tile/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -Makefile -quad_tile.o -quad_tile_so.so diff --git a/images/db/lib/quad_tile/extconf.rb b/images/db/lib/quad_tile/extconf.rb deleted file mode 100644 index 2264a3e0..00000000 --- a/images/db/lib/quad_tile/extconf.rb +++ /dev/null @@ -1,5 +0,0 @@ -require "mkmf" - -with_cflags("-std=c99 #{$CFLAGS}") do - create_makefile("quad_tile_so") -end diff --git a/images/db/lib/quad_tile/quad_tile.c b/images/db/lib/quad_tile/quad_tile.c deleted file mode 100644 index cd45e6e7..00000000 --- a/images/db/lib/quad_tile/quad_tile.c +++ /dev/null @@ -1,148 +0,0 @@ -#include "ruby.h" -#include "quad_tile.h" - -typedef struct { - unsigned int *tilev; - unsigned int tilec; -} tilelist_t; - -static tilelist_t tilelist_for_area(unsigned int minx, unsigned int miny, unsigned int maxx, unsigned int maxy) -{ - unsigned int x; - unsigned int y; - tilelist_t tl; - unsigned int maxtilec; - - maxtilec = 256; - - tl.tilev = malloc(maxtilec * sizeof(unsigned int)); - tl.tilec = 0; - - for (x = minx; x <= maxx; x++) - { - for (y = miny; y <= maxy; y++) - { - if (tl.tilec == maxtilec) - { - maxtilec = maxtilec * 2; - - tl.tilev = realloc(tl.tilev, maxtilec * sizeof(unsigned int)); - } - - tl.tilev[tl.tilec++] = xy2tile(x, y); - } - } - - return tl; -} - -static int tile_compare(const void *ap, const void *bp) -{ - unsigned int a = *(unsigned int *)ap; - unsigned int b = *(unsigned int *)bp; - - if (a < b) - { - return -1; - } - else if (a > b) - { - return 1; - } - else - { - return 0; - } -} - -static VALUE tile_for_point(VALUE self, VALUE lat, VALUE lon) -{ - unsigned int x = lon2x(NUM2DBL(lon)); - unsigned int y = lat2y(NUM2DBL(lat)); - - return UINT2NUM(xy2tile(x, y)); -} - -static VALUE tiles_for_area(VALUE self, VALUE bbox) -{ - unsigned int minx = lon2x(NUM2DBL(rb_iv_get(bbox, "@min_lon"))); - unsigned int maxx = lon2x(NUM2DBL(rb_iv_get(bbox, "@max_lon"))); - unsigned int miny = lat2y(NUM2DBL(rb_iv_get(bbox, "@min_lat"))); - unsigned int maxy = lat2y(NUM2DBL(rb_iv_get(bbox, "@max_lat"))); - tilelist_t tl = tilelist_for_area(minx, miny, maxx, maxy); - VALUE tiles = rb_ary_new(); - unsigned int t; - - for (t = 0; t < tl.tilec; t++) - { - rb_ary_push(tiles, UINT2NUM(tl.tilev[tl.tilec])); - } - - free(tl.tilev); - - return tiles; -} - -static VALUE tile_for_xy(VALUE self, VALUE x, VALUE y) -{ - return UINT2NUM(xy2tile(NUM2UINT(x), NUM2UINT(y))); -} - -static VALUE iterate_tiles_for_area(VALUE self, VALUE bbox) -{ - unsigned int minx = lon2x(NUM2DBL(rb_iv_get(bbox, "@min_lon"))); - unsigned int maxx = lon2x(NUM2DBL(rb_iv_get(bbox, "@max_lon"))); - unsigned int miny = lat2y(NUM2DBL(rb_iv_get(bbox, "@min_lat"))); - unsigned int maxy = lat2y(NUM2DBL(rb_iv_get(bbox, "@max_lat"))); - tilelist_t tl = tilelist_for_area(minx, miny, maxx, maxy); - - if (tl.tilec > 0) - { - unsigned int first; - unsigned int last; - unsigned int t; - VALUE a = rb_ary_new(); - - qsort(tl.tilev, tl.tilec, sizeof(unsigned int), tile_compare); - - first = last = tl.tilev[0]; - - for (t = 1; t < tl.tilec; t++) - { - unsigned int tile = tl.tilev[t]; - - if (tile == last + 1) - { - last = tile; - } - else - { - rb_ary_store(a, 0, UINT2NUM(first)); - rb_ary_store(a, 1, UINT2NUM(last)); - rb_yield(a); - - first = last = tile; - } - } - - rb_ary_store(a, 0, UINT2NUM(first)); - rb_ary_store(a, 1, UINT2NUM(last)); - rb_yield(a); - } - - free(tl.tilev); - - return Qnil; -} - -void Init_quad_tile_so(void) -{ - VALUE m = rb_define_module("QuadTile"); - - rb_define_module_function(m, "tile_for_point", tile_for_point, 2); - rb_define_module_function(m, "tiles_for_area", tiles_for_area, 1); - rb_define_module_function(m, "tile_for_xy", tile_for_xy, 2); - rb_define_module_function(m, "iterate_tiles_for_area", iterate_tiles_for_area, 1); - - return; -} diff --git a/images/db/lib/quad_tile/quad_tile.h b/images/db/lib/quad_tile/quad_tile.h deleted file mode 100644 index f868ff51..00000000 --- a/images/db/lib/quad_tile/quad_tile.h +++ /dev/null @@ -1,25 +0,0 @@ -#include - -inline unsigned int xy2tile(unsigned int x, unsigned int y) -{ - unsigned int tile = 0; - int i; - - for (i = 15; i >= 0; i--) - { - tile = (tile << 1) | ((x >> i) & 1); - tile = (tile << 1) | ((y >> i) & 1); - } - - return tile; -} - -inline unsigned int lon2x(double lon) -{ - return round((lon + 180.0) * 65535.0 / 360.0); -} - -inline unsigned int lat2y(double lat) -{ - return round((lat + 90.0) * 65535.0 / 180.0); -} diff --git a/images/full-history/Dockerfile b/images/full-history/Dockerfile index 3c608c1e..330efd33 100644 --- a/images/full-history/Dockerfile +++ b/images/full-history/Dockerfile @@ -1,4 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99 +FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.984.h5103747 VOLUME /mnt/data COPY ./start.sh / diff --git a/images/full-history/start.sh b/images/full-history/start.sh index b4cc0dbb..43b4e447 100755 --- a/images/full-history/start.sh +++ b/images/full-history/start.sh @@ -27,54 +27,60 @@ fi # =============================== -# Download db .dump file +# Download db .dump file # =============================== download_dump_file() { echo "Downloading db .dump file from cloud..." + local temp_dump_file="$dumpFile.tmp" + local actual_dump_url="" + if [ "$CLOUDPROVIDER" == "aws" ]; then if [[ "$DUMP_CLOUD_URL" == *.txt ]]; then temp_txt="$VOLUME_DIR/tmp_dump_url.txt" aws s3 cp "$DUMP_CLOUD_URL" "$temp_txt" - # Get the first line (S3 URL to the .dump or .dump.gz file) - first_line=$(head -n 1 "$temp_txt") - echo "Found dump URL in txt: $first_line" - - # Set dump file name based on extension - if [[ "$first_line" == *.gz ]]; then - dumpFile="${dumpFile}.gz" - fi - - aws s3 cp "$first_line" "$dumpFile" - if [[ "$dumpFile" == *.gz ]]; then - echo "Decompressing gzip file..." - gunzip -f "$dumpFile" - dumpFile="${dumpFile%.gz}" - fi + # Get the first line (S3 URL to the .dump file) + actual_dump_url=$(head -n 1 "$temp_txt") + echo "Found dump URL in txt: $actual_dump_url" + + aws s3 cp "$actual_dump_url" "$temp_dump_file" rm -f "$temp_txt" else - # Set dump file name based on extension - if [[ "$DUMP_CLOUD_URL" == *.gz ]]; then - dumpFile="${dumpFile}.gz" - fi - aws s3 cp "$DUMP_CLOUD_URL" "$dumpFile" - if [[ "$dumpFile" == *.gz ]]; then - echo "Decompressing gzip file..." - gunzip -f "$dumpFile" - dumpFile="${dumpFile%.gz}" - fi + actual_dump_url="$DUMP_CLOUD_URL" + aws s3 cp "$DUMP_CLOUD_URL" "$temp_dump_file" fi elif [ "$CLOUDPROVIDER" == "gcp" ]; then - gsutil cp "$DUMP_CLOUD_URL" "$dumpFile" + actual_dump_url="$DUMP_CLOUD_URL" + gsutil cp "$DUMP_CLOUD_URL" "$temp_dump_file" else echo "Unsupported CLOUDPROVIDER: $CLOUDPROVIDER" exit 1 fi - echo "Dump file ready at: $dumpFile" + # Check if downloaded file is gzip compressed and decompress if needed + # Check by file extension, file type, or magic bytes + local is_gzip=false + if [[ "$actual_dump_url" == *.gz ]] || [[ "$temp_dump_file" == *.gz ]]; then + is_gzip=true + elif command -v file >/dev/null 2>&1 && file "$temp_dump_file" 2>/dev/null | grep -q "gzip compressed"; then + is_gzip=true + elif head -c 2 "$temp_dump_file" 2>/dev/null | od -An -tx1 | grep -q "1f 8b"; then + # Check for gzip magic bytes (1f 8b) + is_gzip=true + fi + + if [ "$is_gzip" = true ]; then + echo "Detected gzip compressed dump file, decompressing..." + gunzip -c "$temp_dump_file" > "$dumpFile" + rm -f "$temp_dump_file" + else + mv "$temp_dump_file" "$dumpFile" + fi + + echo "Dump file ready at: $dumpFile (PostgreSQL 17 compatible)" } # =============================== diff --git a/images/osm-processor/Dockerfile b/images/osm-processor/Dockerfile index 70754d0c..04917ffc 100644 --- a/images/osm-processor/Dockerfile +++ b/images/osm-processor/Dockerfile @@ -32,6 +32,14 @@ ENV workdir /mnt/data WORKDIR $workdir RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + && echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ osmosis \ @@ -39,13 +47,12 @@ RUN set -ex \ awscli \ gsutil \ azure-cli \ - postgresql-client \ + postgresql-client-17 \ rsync \ pyosmium \ tmux \ zsh \ git \ - curl \ libxml2 \ libboost-filesystem1.74.0 \ libboost-program-options1.74.0 \ diff --git a/images/planet-dump/Dockerfile b/images/planet-dump/Dockerfile index a7352b8d..585ae7d6 100644 --- a/images/planet-dump/Dockerfile +++ b/images/planet-dump/Dockerfile @@ -1,5 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99 - +FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.984.h5103747 VOLUME /mnt/data COPY ./start.sh / -CMD /start.sh \ No newline at end of file +CMD /start.sh diff --git a/images/planet-dump/start.sh b/images/planet-dump/start.sh index 7b0902e9..e79d9f7a 100755 --- a/images/planet-dump/start.sh +++ b/images/planet-dump/start.sh @@ -25,54 +25,60 @@ fi # =============================== -# Download db .dump file +# Download db .dump file # =============================== download_dump_file() { echo "Downloading db .dump file from cloud..." + local temp_dump_file="$dumpFile.tmp" + local actual_dump_url="" + if [ "$CLOUDPROVIDER" == "aws" ]; then if [[ "$DUMP_CLOUD_URL" == *.txt ]]; then temp_txt="$VOLUME_DIR/tmp_dump_url.txt" aws s3 cp "$DUMP_CLOUD_URL" "$temp_txt" - # Get the first line (S3 URL to the .dump or .dump.gz file) - first_line=$(head -n 1 "$temp_txt") - echo "Found dump URL in txt: $first_line" - - # Set dump file name based on extension - if [[ "$first_line" == *.gz ]]; then - dumpFile="${dumpFile}.gz" - fi - - aws s3 cp "$first_line" "$dumpFile" - if [[ "$dumpFile" == *.gz ]]; then - echo "Decompressing gzip file..." - gunzip -f "$dumpFile" - dumpFile="${dumpFile%.gz}" - fi + # Get the first line (S3 URL to the .dump file) + actual_dump_url=$(head -n 1 "$temp_txt") + echo "Found dump URL in txt: $actual_dump_url" + + aws s3 cp "$actual_dump_url" "$temp_dump_file" rm -f "$temp_txt" else - # Set dump file name based on extension - if [[ "$DUMP_CLOUD_URL" == *.gz ]]; then - dumpFile="${dumpFile}.gz" - fi - aws s3 cp "$DUMP_CLOUD_URL" "$dumpFile" - if [[ "$dumpFile" == *.gz ]]; then - echo "Decompressing gzip file..." - gunzip -f "$dumpFile" - dumpFile="${dumpFile%.gz}" - fi + actual_dump_url="$DUMP_CLOUD_URL" + aws s3 cp "$DUMP_CLOUD_URL" "$temp_dump_file" fi elif [ "$CLOUDPROVIDER" == "gcp" ]; then - gsutil cp "$DUMP_CLOUD_URL" "$dumpFile" + actual_dump_url="$DUMP_CLOUD_URL" + gsutil cp "$DUMP_CLOUD_URL" "$temp_dump_file" else echo "Unsupported CLOUDPROVIDER: $CLOUDPROVIDER" exit 1 fi - echo "Dump file ready at: $dumpFile" + # Check if downloaded file is gzip compressed and decompress if needed + # Check by file extension, file type, or magic bytes + local is_gzip=false + if [[ "$actual_dump_url" == *.gz ]] || [[ "$temp_dump_file" == *.gz ]]; then + is_gzip=true + elif command -v file >/dev/null 2>&1 && file "$temp_dump_file" 2>/dev/null | grep -q "gzip compressed"; then + is_gzip=true + elif head -c 2 "$temp_dump_file" 2>/dev/null | od -An -tx1 | grep -q "1f 8b"; then + # Check for gzip magic bytes (1f 8b) + is_gzip=true + fi + + if [ "$is_gzip" = true ]; then + echo "Detected gzip compressed dump file, decompressing..." + gunzip -c "$temp_dump_file" > "$dumpFile" + rm -f "$temp_dump_file" + else + mv "$temp_dump_file" "$dumpFile" + fi + + echo "Dump file ready at: $dumpFile (PostgreSQL 17 compatible)" } diff --git a/images/replication-job/Dockerfile b/images/replication-job/Dockerfile index 843d7c1a..6ba15f6e 100644 --- a/images/replication-job/Dockerfile +++ b/images/replication-job/Dockerfile @@ -1,21 +1,55 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574 +FROM debian:bookworm AS builder -RUN apt-get update && \ - apt-get install -y \ - nginx \ - python3-pip \ - python3-venv \ - procps \ - curl && \ - rm -rf /var/lib/apt/lists/* +# Setup PostgreSQL repository and install build dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl gnupg \ + && . /etc/os-release \ + && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + git build-essential cmake pandoc \ + libosmium2-dev libprotozero-dev libyaml-cpp-dev \ + libpqxx-6.4 libpqxx-dev \ + libboost-program-options-dev libboost-system-dev libboost-filesystem-dev \ + postgresql-server-dev-17 libbz2-dev libexpat1-dev zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" -RUN pip install --no-cache-dir boto3 +# Build osmdbt +RUN git clone https://github.com/openstreetmap/osmdbt.git /osmdbt \ + && cd /osmdbt && git checkout v0.9 \ + && mkdir -p build && cd build \ + && cmake -DBUILD_PLUGIN=OFF .. && make -COPY ./*.sh / -COPY monitoring.py / +RUN mkdir -p /tmp/runtime-libs \ + && find /usr/lib/x86_64-linux-gnu /usr/local/lib -name "libosmium*.so*" -exec cp {} /tmp/runtime-libs/ \; 2>/dev/null || true \ + && find /usr/lib/x86_64-linux-gnu /usr/local/lib -name "libprotozero*.so*" -exec cp {} /tmp/runtime-libs/ \; 2>/dev/null || true -WORKDIR /mnt/data +# ============================================================================ +# Runtime stage +# ============================================================================ +FROM debian:bookworm -CMD /start.sh +# Install runtime dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl gzip bash procps postgresql-client \ + openjdk-17-jre-headless osmosis awscli \ + libpqxx-6.4 libyaml-cpp0.7 \ + libboost-program-options1.74.0 libboost-system1.74.0 libboost-filesystem1.74.0 \ + libbz2-1.0 libexpat1 zlib1g \ + && rm -rf /var/lib/apt/lists/* + +# Copy binaries and third-party libraries (libosmium, libprotozero from builder) +COPY --from=builder /osmdbt/build /osmdbt/build +COPY --from=builder /usr/local/bin /usr/local/bin +COPY --from=builder /tmp/runtime-libs/ /usr/local/lib/ + +RUN ldconfig + +ENV PATH="/osmdbt/build:/usr/local/bin:$PATH" + +COPY start.sh /start.sh +COPY liveness.sh /liveness.sh +ENTRYPOINT ["/bin/bash","-c"] +CMD ["/start.sh"] diff --git a/images/replication-job/README.md b/images/replication-job/README.md index 86196843..01e29b4a 100644 --- a/images/replication-job/README.md +++ b/images/replication-job/README.md @@ -1,29 +1,109 @@ -### Delta replications job container +# OSM Replication Job Container -This contain is responsible for creating the delta replication files it may be set up by minute, hour, etc. Those replications files will be uploaded to a repository in AWS or Google Storage, depends on what you are using. +This container is responsible for creating OSM delta replication files using [osmdbt](https://github.com/openstreetmap/osmdbt) tools. It generates replication files every minute and uploads them to S3 with integrity verification. -### Configuration +## What it does -In order to run this container we need environment variables, these can be found in the following files👇: +The container runs a continuous replication process that: -- [.env.db.example](./../../envs/.env.db.example) -- [.env.db-utils.example](./../../envs/.env.db-utils.example) -- [.env.cloudprovider.example](./../../envs/.env.cloudprovider.example) +1. **Executes replication cycle every minute**: + - Runs `osmdbt-get-log` to fetch changes from PostgreSQL logical replication + - Runs `osmdbt-create-diff` to generate `.osc.gz` files and `state.txt` files + +2. **Uploads files to S3**: + - Uploads `.osc.gz` files (e.g., `870.osc.gz`) + - Uploads general `state.txt` (controls replication sequence) + - Uploads specific `state.txt` files (e.g., `870.state.txt` for `870.osc.gz`) + - All files are verified for integrity before upload + +3. **Manages replication state**: + - Uses local `state.txt` to control replication sequence + - Recovers `state.txt` from S3 on container restart + - Continues from the last known sequence automatically + +4. **Error handling and monitoring**: + - Verifies file integrity (gzip test, size checks) + - Sends Slack notifications for errors and corruption + - Automatically cleans up incomplete/orphaned files (`.lock`, `.log`, corrupted files) + - Removes orphaned state files without matching `.osc.gz` files + +5. **Self-healing**: + - Detects and removes corrupted files + - Regenerates files if corruption is detected + - Maintains sequence continuity + +## Technology Stack + +- **osmdbt v0.9**: OSM Database Replication Tools from OpenStreetMap +- **Base Image**: Debian Bookworm (for library version consistency) +- **PostgreSQL**: Uses logical replication slot for change tracking +- **AWS S3**: For storing replication files +- **Slack**: Optional notifications for errors + +## Configuration + +### Required Environment Variables + +The container requires environment variables from these files: + +- [.env.db.example](../../envs/.env.db.example) - Database connection +- [.env.db-utils.example](../../envs/.env.db-utils.example) - Database utilities +- [.env.cloudprovider.example](../../envs/.env.cloudprovider.example) - Cloud storage configuration **Note**: Rename the above files as `.env.db`, `.env.db-utils` and `.env.cloudprovider` -#### Running replication-job container +### Key Environment Variables + +#### Database Configuration +- `POSTGRES_HOST` - PostgreSQL hostname +- `POSTGRES_PORT` - PostgreSQL port (default: 5432) +- `POSTGRES_DB` - Database name +- `POSTGRES_USER` - Database user (must have replication and SELECT permissions) +- `POSTGRES_PASSWORD` - Database password +- `REPLICATION_SLOT` - Logical replication slot name (default: `osm_repl`) + +**Important**: The database user must have the following permissions: +- `REPLICATION` privilege (to read from logical replication slot) +- `SELECT` permission on the `changesets` table (required by `osmdbt-create-diff`) + +To grant these permissions, run as a database administrator: +```sql +-- Create user with replication privilege +CREATE ROLE osmdbt_user WITH REPLICATION LOGIN PASSWORD 'your_password'; + +-- Grant SELECT on changesets table +GRANT SELECT ON TABLE changesets TO osmdbt_user; +``` + +#### S3 Configuration +- `CLOUDPROVIDER` - Cloud provider (default: `aws`) +- `AWS_S3_BUCKET` - S3 bucket name for replication files +- `REPLICATION_FOLDER` - Folder path in S3 bucket (default: `replication`) + +#### Slack Notifications (Optional) +- `ENABLE_SEND_SLACK_MESSAGE` - Enable Slack notifications (default: `false`) +- `SLACK_WEBHOOK_URL` - Slack webhook URL for notifications +- `ENVIROMENT` - Environment name for notifications (e.g., `production`, `staging`) + +#### Working Directory +- `WORKING_DIRECTORY` - Working directory path (default: `/mnt/data`) + +## Running the Container + +### Using Docker Compose + +```sh +docker-compose run replication-job +``` + +### Using Docker ```sh - # Docker compose - docker-compose run replication-job - - # Docker - docker run \ - --env-file ./envs/.env.db \ - --env-file ./envs/.env.replication-job \ - --env-file ./envs/.env.cloudprovider \ - -v ${PWD}/data/replication-job-data:/mnt/data \ - --network osm-seed_default \ - -it osmseed-replication-job:v1 +docker run \ + --env-file ./envs/.env.db \ + --env-file ./envs/.env.replication-job \ + --env-file ./envs/.env.cloudprovider \ + -v ${PWD}/data/replication-job-data:/mnt/data \ + --network osm-seed_default \ + -it osmseed-replication-job:v1 ``` diff --git a/images/replication-job/liveness.sh b/images/replication-job/liveness.sh index 7b230987..4f8d252e 100755 --- a/images/replication-job/liveness.sh +++ b/images/replication-job/liveness.sh @@ -1,44 +1,40 @@ #!/usr/bin/env bash -# Script to check if Osmosis (Java) is running and also check how old processed_files.log is. -# If processed_files.log is older than MAX_AGE_MINUTES, then kill Osmosis. +# Liveness check for OSM replication process +# Checks if main process is running, PostgreSQL connection, and if log file is being updated +# Exit codes: 0=healthy, 1=process not running or DB unreachable, 2=process stuck -LOG_FILE="/mnt/data/processed_files.log" -MAX_AGE_MINUTES=10 +LOG_FILE="${WORKING_DIRECTORY:-/mnt/data}/logs/processed_files.log" +MAX_AGE_MINUTES="${MAX_LOG_AGE_MINUTES:-10}" -get_file_age_in_minutes() { - local file="$1" - if [ ! -f "$file" ]; then - echo 999999 - return - fi - local now - local mtime - now=$(date +%s) - mtime=$(stat -c %Y "$file") - local diff=$(( (now - mtime) / 60 )) - echo "$diff" -} +# Check if main process (start.sh) is running +if ! pgrep -f "start.sh" >/dev/null 2>&1; then + echo "Main process (start.sh) is not running!" + exit 1 +fi -# Check if Osmosis (Java) is running -OSMOSIS_COUNT=$(ps -ef | grep -E 'java.*osmosis' | grep -v grep | wc -l) +# Check PostgreSQL connection +export PGPASSWORD="${POSTGRES_PASSWORD:-}" +if ! pg_isready -h "${POSTGRES_HOST:-localhost}" -p "${POSTGRES_PORT:-5432}" -U "${POSTGRES_USER:-osm}" -d "${POSTGRES_DB:-osm}" >/dev/null 2>&1; then + echo "PostgreSQL is not reachable!" + exit 1 +fi -if [ "$OSMOSIS_COUNT" -ge 1 ]; then - echo "Osmosis is running." - # Check how old the processed_files.log file is - file_age=$(get_file_age_in_minutes "$LOG_FILE") - echo "processed_files.log file age in minutes: $file_age" - if [ "$file_age" -ge "$MAX_AGE_MINUTES" ]; then - echo "processed_files.log is older than $MAX_AGE_MINUTES minutes. Attempting to kill Osmosis and restart the container..." - # Kill the Osmosis process - pkill -f "java.*osmosis" || true - echo "Osmosis is not terminating. Force-killing the container..." - echo "Container force-restart triggered." - exit 2 - else - echo "processed_files.log is not too old. No action needed." - exit 0 - fi -else - echo "Osmosis is not running!" +# Check log file age +if [ ! -f "$LOG_FILE" ]; then + echo "Log file not found: $LOG_FILE" exit 1 fi + +# Get file age in minutes +now=$(date +%s) +mtime=$(stat -c %Y "$LOG_FILE" 2>/dev/null || stat -f %m "$LOG_FILE" 2>/dev/null || echo "0") +file_age=$(( (now - mtime) / 60 )) + +# If log is too old, process might be stuck +if [ "$file_age" -ge "$MAX_AGE_MINUTES" ]; then + echo "Log file is older than $MAX_AGE_MINUTES minutes (age: $file_age min). Process may be stuck." + exit 2 +fi + +echo "Process is healthy (log age: $file_age min)" +exit 0 diff --git a/images/replication-job/start.sh b/images/replication-job/start.sh index e0f0ae92..2e193768 100755 --- a/images/replication-job/start.sh +++ b/images/replication-job/start.sh @@ -1,181 +1,585 @@ #!/usr/bin/env bash -set -e -# osmosis tuning: https://wiki.openstreetmap.org/wiki/Osmosis/Tuning,https://lists.openstreetmap.org/pipermail/talk/2012-October/064771.html -if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then - echo JAVACMD_OPTIONS=\"-server\" >~/.osmosis -else - memory="${MEMORY_JAVACMD_OPTIONS//i/}" - echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis -fi +set -euo pipefail +# ============================================================================ +# OSM Replication Script with osmdbt and S3 +# ============================================================================ +# This script manages OSM replication using osmdbt tools following the +# official osmdbt workflow (https://github.com/openstreetmap/osmdbt): +# +# Initialization: +# - Runs osmdbt-enable-replication to set up replication slot +# - Requires PostgreSQL with wal_level=logical, max_replication_slots >= 1, +# and a user with REPLICATION attribute +# +# Replication Cycle (every minute): +# 1. osmdbt-catchup - Process old log files if any (from crashes) +# 2. osmdbt-get-log - Fetch new changes from PostgreSQL logical replication +# 3. osmdbt-catchup - Update database to new log file +# 4. osmdbt-create-diff - Generate .osc.gz and state.txt files +# +# Additional Features: +# - Uploads .osc.gz and state.txt files to S3 after integrity verification +# * General state.txt (in root directory) - controls replication sequence +# * Specific state.txt files (e.g., 870.state.txt for 870.osc.gz) +# - Uses local state.txt to control replication sequence +# - Recovers state.txt from S3 on restart +# - Sends Slack notifications for errors +# - Cleans up incomplete/orphaned files automatically +# - Allows restoration from S3 by copying state.txt and diffs +# ============================================================================ + +# ---- Configuration Variables ---- +workingDirectory="${WORKING_DIRECTORY:-/mnt/data}" +tmpDirectory="${workingDirectory}/tmp" +runDirectory="${workingDirectory}/run" +changesDir="${workingDirectory}" +logDirectory="${workingDirectory}/logs" # Separate directory for application logs +osmdbtConfig="/osmdbt-config.yaml" + +# Replication variables +REPLICATION_SLOT="${REPLICATION_SLOT:-osm_repl}" +REPLICATION_PLUGIN="osm_logical" + +# S3 Configuration +CLOUDPROVIDER="${CLOUDPROVIDER:-aws}" +AWS_S3_BUCKET="${AWS_S3_BUCKET:-}" +REPLICATION_FOLDER="${REPLICATION_FOLDER:-replication}" + +# Slack Configuration +ENABLE_SLACK="${ENABLE_SEND_SLACK_MESSAGE:-false}" +SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL:-}" slack_message_count=0 -max_slack_messages=2 - -workingDirectory="/mnt/data" -mkdir -p $workingDirectory - -# Remove files that are not required -[ -e /mnt/data/replicate.lock ] && rm -f /mnt/data/replicate.lock -# [ -e /mnt/data/processed_files.log ] && rm -f /mnt/data/processediles.log - -function get_current_state_file() { - # Check if state.txt exist in the workingDirectory, - # in case the file does not exist locally and does not exist in the cloud the replication will start from 0 - if [ ! -f $workingDirectory/state.txt ]; then - echo "File $workingDirectory/state.txt does not exist in local storage" - ### AWS - if [ $CLOUDPROVIDER == "aws" ]; then - aws s3 ls $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt - if [[ $? -eq 0 ]]; then - echo "File exist, let's get it from $CLOUDPROVIDER - $AWS_S3_BUCKET" - aws s3 cp $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt - fi - fi +max_slack_messages_per_hour=10 +slack_reset_time=$(date +%s) + +# Timing +REPLICATION_INTERVAL=60 # seconds (1 minute) +INTEGRITY_CHECK_RETRIES=3 +CLEANUP_INTERVAL=300 # seconds (5 minutes) + +# Process tracking +processed_files_log="${logDirectory}/processed_files.log" +osmdbt_pid_file="${runDirectory}/osmdbt.pid" + +# ---- Directory Setup ---- +mkdir -p "$workingDirectory" "$tmpDirectory" "$runDirectory" "$logDirectory" + +# ---- osmdbt-config.yaml creation ---- +cat < "$osmdbtConfig" +database: + host: ${POSTGRES_HOST:-localhost} + port: ${POSTGRES_PORT:-5432} + dbname: ${POSTGRES_DB:-osm} + user: ${POSTGRES_USER:-osm} + password: ${POSTGRES_PASSWORD} + replication_slot: ${REPLICATION_SLOT} + +log_dir: ${workingDirectory} +changes_dir: ${changesDir} +tmp_dir: ${tmpDirectory} +run_dir: ${runDirectory} +EOF - ### GCP - if [ $CLOUDPROVIDER == "gcp" ]; then - gsutil ls $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt - if [[ $? -eq 0 ]]; then - echo "File exist, let's get it from $CLOUDPROVIDER - $GCP_STORAGE_BUCKET" - gsutil cp $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt +# ============================================================================ +# Function: Enable osmdbt replication (creates slot if needed) +# ============================================================================ +function enable_osmdbt_replication() { + echo "$(date +%F_%H:%M:%S): Enabling osmdbt replication..." + + # Check if replication is already enabled by checking for the slot + export PGPASSWORD="${POSTGRES_PASSWORD}" + local exists=$(psql -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}" \ + -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -t -A -c \ + "SELECT count(*) FROM pg_replication_slots WHERE slot_name='$REPLICATION_SLOT';" 2>/dev/null | tr -d '[:space:]') + + if [ "$exists" = "1" ]; then + echo "$(date +%F_%H:%M:%S): Replication slot '$REPLICATION_SLOT' already exists. Replication should be enabled." + return 0 + fi + + # Use osmdbt-enable-replication to set up replication properly + echo "$(date +%F_%H:%M:%S): Running osmdbt-enable-replication..." + if /osmdbt/build/src/osmdbt-enable-replication -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-enable-replication.log"; then + echo "$(date +%F_%H:%M:%S): Successfully enabled osmdbt replication." + return 0 + else + local error_msg="ERROR: Failed to enable osmdbt replication. Check PostgreSQL configuration (wal_level=logical, max_replication_slots >= 1, user with REPLICATION attribute)." + echo "$(date +%F_%H:%M:%S): $error_msg" + send_slack_message "🚨 ${ENVIROMENT:-production}: $error_msg" + return 1 + fi +} + +# ============================================================================ +# Function: Recover/update state from S3 or local storage +# ============================================================================ +function recover_state_file() { + local state_file="${changesDir}/state.txt" + + # Check if state.txt exists locally + if [ -f "$state_file" ]; then + echo "$(date +%F_%H:%M:%S): Found local state.txt:" + cat "$state_file" + return 0 + fi + + echo "$(date +%F_%H:%M:%S): Local state.txt not found. Attempting to recover from S3..." + + if [ "$CLOUDPROVIDER" == "aws" ] && [ -n "$AWS_S3_BUCKET" ]; then + local s3_state_path="${AWS_S3_BUCKET}/${REPLICATION_FOLDER}/state.txt" + + if aws s3 ls "$s3_state_path" >/dev/null 2>&1; then + echo "$(date +%F_%H:%M:%S): Downloading state.txt from S3: $s3_state_path" + if aws s3 cp "$s3_state_path" "$state_file"; then + echo "$(date +%F_%H:%M:%S): Successfully recovered state.txt from S3:" + cat "$state_file" + return 0 + else + echo "$(date +%F_%H:%M:%S): WARNING: Failed to download state.txt from S3" fi + else + echo "$(date +%F_%H:%M:%S): state.txt not found in S3. Starting fresh replication." fi + fi + + echo "$(date +%F_%H:%M:%S): No state.txt found. Will start replication from beginning." + return 0 +} - ### Azure - if [ $CLOUDPROVIDER == "azure" ]; then - state_file_exists=$(az storage blob exists --container-name $AZURE_CONTAINER_NAME --name $REPLICATION_FOLDER/state.txt --query="exists") - if [[ $state_file_exists=="true" ]]; then - echo "File exist, let's get it from $CLOUDPROVIDER - $AZURE_CONTAINER_NAME" - az storage blob download \ - --container-name $AZURE_CONTAINER_NAME \ - --name $REPLICATION_FOLDER/state.txt \ - --file $workingDirectory/state.txt --query="name" +# ============================================================================ +# Function: Verify file integrity +# ============================================================================ +function verify_file_integrity() { + local file="$1" + local file_type="$2" # "osc" or "state" + local retries=0 + + while [ $retries -lt $INTEGRITY_CHECK_RETRIES ]; do + if [ "$file_type" == "osc" ]; then + # Verify .osc.gz file + if [ ! -f "$file" ]; then + echo "$(date +%F_%H:%M:%S): ERROR: File does not exist: $file" + return 1 + fi + + # Check if file is complete (not truncated) + if ! gzip -t "$file" 2>/dev/null; then + echo "$(date +%F_%H:%M:%S): WARNING: gzip integrity check failed for $file (attempt $((retries + 1))/$INTEGRITY_CHECK_RETRIES)" + retries=$((retries + 1)) + sleep 2 + continue fi + + # Check file size (should be > 0) + local file_size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null || echo "0") + if [ "$file_size" -eq 0 ]; then + echo "$(date +%F_%H:%M:%S): WARNING: File is empty: $file" + return 1 + fi + + echo "$(date +%F_%H:%M:%S): Integrity check passed for $file (size: $file_size bytes)" + return 0 + + elif [ "$file_type" == "state" ]; then + # Verify state.txt file + if [ ! -f "$file" ]; then + echo "$(date +%F_%H:%M:%S): ERROR: state.txt does not exist: $file" + return 1 + fi + + # Check if state.txt has required fields + if ! grep -q "sequenceNumber=" "$file" 2>/dev/null; then + echo "$(date +%F_%H:%M:%S): WARNING: state.txt missing sequenceNumber (attempt $((retries + 1))/$INTEGRITY_CHECK_RETRIES)" + retries=$((retries + 1)) + sleep 1 + continue + fi + + echo "$(date +%F_%H:%M:%S): Integrity check passed for state.txt" + return 0 fi + done + + echo "$(date +%F_%H:%M:%S): ERROR: Integrity check failed after $INTEGRITY_CHECK_RETRIES attempts for $file" + return 1 +} + +# ============================================================================ +# Function: Upload files to S3 with integrity verification +# ============================================================================ +function upload_file_to_s3() { + local local_file="$1" + local file_type="$2" # "osc" or "state" + + if [ "$CLOUDPROVIDER" != "aws" ] || [ -z "$AWS_S3_BUCKET" ]; then + echo "$(date +%F_%H:%M:%S): WARNING: S3 upload skipped (CLOUDPROVIDER=$CLOUDPROVIDER, AWS_S3_BUCKET not set)" + return 0 + fi + + # Verify integrity before upload + if ! verify_file_integrity "$local_file" "$file_type"; then + local error_msg="🚨 ${ENVIROMENT:-production}: Integrity check failed for $local_file. File will not be uploaded." + echo "$(date +%F_%H:%M:%S): $error_msg" + return 1 + fi + + # Determine S3 path preserving directory structure + # Get relative path from changesDir (e.g., /mnt/data/000/871/309.osc.gz -> 000/871/309.osc.gz) + local relative_path="${local_file#${changesDir}/}" + # Remove leading slash if changesDir doesn't end with one + relative_path="${relative_path#/}" + local s3_path="${AWS_S3_BUCKET}/${REPLICATION_FOLDER}/${relative_path}" + + echo "$(date +%F_%H:%M:%S): Uploading $local_file to S3: $s3_path" + + if aws s3 cp "$local_file" "$s3_path" --acl public-read; then + echo "$(date +%F_%H:%M:%S): Successfully uploaded $relative_path to S3" + return 0 else - echo "File $workingDirectory/state.txt exist in local storage" - echo "File $workingDirectory/state.txt content:" - cat $workingDirectory/state.txt + local error_msg="🚨 ${ENVIROMENT:-production}: Failed to upload $relative_path to S3" + echo "$(date +%F_%H:%M:%S): $error_msg" + return 1 fi } -function upload_file_cloud() { - # Upload files to cloud provider - local local_file="$1" - local cloud_file="$REPLICATION_FOLDER/${local_file#*"$workingDirectory/"}" - echo "$(date +%F_%H:%M:%S): Upload file $local_file to ...$CLOUDPROVIDER...$cloud_file" - if [ "$CLOUDPROVIDER" == "aws" ]; then - aws s3 cp "$local_file" "$AWS_S3_BUCKET/$cloud_file" --acl public-read - elif [ "$CLOUDPROVIDER" == "gcp" ]; then - gsutil cp -a public-read "$local_file" "$GCP_STORAGE_BUCKET/$cloud_file" - elif [ "$CLOUDPROVIDER" == "azure" ]; then - az storage blob upload \ - --container-name "$AZURE_CONTAINER_NAME" \ - --file "$local_file" \ - --name "$cloud_file" \ - --output none +# ============================================================================ +# Function: Upload all new replication files +# ============================================================================ +function upload_replication_files() { + local osc_file="$1" + local general_state_file="${changesDir}/state.txt" + + # Extract base name and directory from osc file + # Handle paths like /mnt/data/000/871/305.osc.gz -> /mnt/data/000/871/305.state.txt + local osc_dir=$(dirname "$osc_file") + local osc_basename=$(basename "$osc_file" .osc.gz) + local specific_state_file="${osc_dir}/${osc_basename}.state.txt" + + local upload_success=true + + # Upload .osc.gz file + if [ -f "$osc_file" ]; then + if ! upload_file_to_s3 "$osc_file" "osc"; then + upload_success=false + fi + else + echo "$(date +%F_%H:%M:%S): WARNING: OSC file not found: $osc_file" + upload_success=false + fi + + # Upload specific state.txt (e.g., 870.state.txt for 870.osc.gz) + if [ -f "$specific_state_file" ]; then + echo "$(date +%F_%H:%M:%S): Found specific state file: $specific_state_file" + if ! upload_file_to_s3 "$specific_state_file" "state"; then + upload_success=false + fi + else + echo "$(date +%F_%H:%M:%S): WARNING: Specific state.txt not found: $specific_state_file" + # This is not necessarily a failure, as some setups may not generate specific state files + fi + + # Upload general state.txt (always upload this one) + if [ -f "$general_state_file" ]; then + if ! upload_file_to_s3 "$general_state_file" "state"; then + upload_success=false + fi + else + echo "$(date +%F_%H:%M:%S): WARNING: General state.txt not found: $general_state_file" + upload_success=false + fi + + if [ "$upload_success" = true ]; then + # Log successful processing + echo "$(date +%F_%H:%M:%S): $osc_file: SUCCESS" >> "$processed_files_log" + return 0 + else + # Log failure + echo "$(date +%F_%H:%M:%S): $osc_file: FAILURE" >> "$processed_files_log" + return 1 fi } +# ============================================================================ +# Function: Send Slack notifications +# ============================================================================ function send_slack_message() { - # Check if Slack messaging is enabled - if [ "${ENABLE_SEND_SLACK_MESSAGE}" != "true" ]; then - echo "Slack messaging is disabled. Set ENABLE_SEND_SLACK_MESSAGE to true to enable." - return + if [ "$ENABLE_SLACK" != "true" ]; then + return 0 fi - - # Check if the Slack webhook URL is set - if [ -z "${SLACK_WEBHOOK_URL}" ]; then - echo "SLACK_WEBHOOK_URL is not set. Unable to send message to Slack." + + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "$(date +%F_%H:%M:%S): WARNING: SLACK_WEBHOOK_URL not set. Cannot send Slack message." return 1 fi - - # Limit Slack message count to 3 - if [ "$slack_message_count" -ge "$max_slack_messages" ]; then - echo "Max Slack messages limit reached. No further messages will be sent." - return + + # Reset counter every hour + local current_time=$(date +%s) + if [ $((current_time - slack_reset_time)) -ge 3600 ]; then + slack_message_count=0 + slack_reset_time=$current_time fi - + + if [ "$slack_message_count" -ge "$max_slack_messages_per_hour" ]; then + echo "$(date +%F_%H:%M:%S): Max Slack messages limit reached ($max_slack_messages_per_hour/hour). Message not sent." + return 0 + fi + local message="$1" - curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"$message\"}" "$SLACK_WEBHOOK_URL" - echo "Message sent to Slack: $message" - slack_message_count=$((slack_message_count + 1)) + local timestamp=$(date +%F_%H:%M:%S) + local full_message="[OSM Replication] $timestamp - $message" + + if curl -X POST -H 'Content-type: application/json' \ + --data "{\"text\": \"$full_message\"}" \ + "$SLACK_WEBHOOK_URL" >/dev/null 2>&1; then + echo "$(date +%F_%H:%M:%S): Slack notification sent: $message" + slack_message_count=$((slack_message_count + 1)) + return 0 + else + echo "$(date +%F_%H:%M:%S): WARNING: Failed to send Slack notification" + return 1 + fi } +# ============================================================================ +# Function: Clean up incomplete/orphaned files +# ============================================================================ +function cleanup_orphaned_files() { + echo "$(date +%F_%H:%M:%S): Cleaning up orphaned files..." + + local cleaned=0 + + # Remove lock files + find "$workingDirectory" -name "*.lock" -type f -mmin +10 -delete && cleaned=$((cleaned + 1)) + find "$runDirectory" -name "*.lock" -type f -mmin +10 -delete && cleaned=$((cleaned + 1)) + + # Remove old application log files (keep only recent ones) + find "$logDirectory" -name "*.log" -type f -mtime +7 -delete && cleaned=$((cleaned + 1)) + + # Remove incomplete .osc.gz files (0 bytes or corrupted) + while IFS= read -r -d '' file; do + local size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null || echo "0") + if [ "$size" -eq 0 ] || ! gzip -t "$file" 2>/dev/null; then + echo "$(date +%F_%H:%M:%S): Removing corrupted/incomplete file: $file" + rm -f "$file" + cleaned=$((cleaned + 1)) + fi + done < <(find "$changesDir" -name "*.osc.gz" -type f -print0 2>/dev/null) + + # Remove orphaned specific state files (not matching any .osc.gz) + # Note: We preserve the general state.txt file + while IFS= read -r -d '' state_file; do + local filename=$(basename "$state_file") + # Skip the general state.txt file + if [ "$filename" = "state.txt" ]; then + continue + fi + + # Extract base name (e.g., "870.state.txt" -> "870") + local base=$(basename "$state_file" .state.txt) + local osc_file="${changesDir}/${base}.osc.gz" + + if [ ! -f "$osc_file" ]; then + echo "$(date +%F_%H:%M:%S): Removing orphaned specific state file: $state_file" + rm -f "$state_file" + cleaned=$((cleaned + 1)) + fi + done < <(find "$changesDir" -name "*.state.txt" -type f -print0 2>/dev/null) + + if [ $cleaned -gt 0 ]; then + echo "$(date +%F_%H:%M:%S): Cleaned up $cleaned orphaned/incomplete file(s)" + else + echo "$(date +%F_%H:%M:%S): No orphaned files found" + fi +} -function monitor_minute_replication() { - # Function to handle continuous monitoring, minute replication, and sequential upload to cloud provider - # Directory to store a log of the last processed file - processed_files_log="$workingDirectory/processed_files.log" - max_log_size_mb=1 +# ============================================================================ +# Function: Verify sequence continuity +# ============================================================================ +function verify_sequence_continuity() { + local state_file="${changesDir}/state.txt" + + if [ ! -f "$state_file" ]; then + echo "$(date +%F_%H:%M:%S): WARNING: state.txt not found for sequence verification" + return 1 + fi + + local current_seq=$(grep "sequenceNumber=" "$state_file" | sed 's/.*sequenceNumber=\([0-9]*\).*/\1/') + + if [ -z "$current_seq" ]; then + echo "$(date +%F_%H:%M:%S): ERROR: Could not extract sequence number from state.txt" + return 1 + fi + + echo "$(date +%F_%H:%M:%S): Current sequence number: $current_seq" + return 0 +} - while true; do - if [ -e "$processed_files_log" ]; then - log_size=$(du -m "$processed_files_log" | cut -f1) - if [ "$log_size" -gt "$max_log_size_mb" ]; then - echo $(date +%F_%H:%M:%S)": Cleaning processed_files_log..." >"$processed_files_log" - fi - # Find new .gz files created within the last minute - for local_minute_file in $(find $workingDirectory/ -name "*.gz" -cmin -1); do - if [ -f "$local_minute_file" ]; then - echo "Processing $local_minute_file..." - # Ensure the file is uploaded only once - if ! grep -q "$local_minute_file: SUCCESS" "$processed_files_log" && ! grep -q "$local_minute_file: FAILURE" "$processed_files_log"; then - # Verify gz file integrity - if gzip -t "$local_minute_file" 2>/dev/null; then - # Upload the file sequentially - upload_file_cloud $local_minute_file - local_state_file="${local_minute_file%.osc.gz}.state.txt" - upload_file_cloud $local_state_file - echo "$local_minute_file: SUCCESS" >>"$processed_files_log" - # Upload and update state.txt after successful upload - upload_file_cloud "$workingDirectory/state.txt" - else - echo $(date +%F_%H:%M:%S)": $local_minute_file is corrupted and will not be uploaded." >>"$processed_files_log" - echo "$local_minute_file: FAILURE" >>"$processed_files_log" - # Ensure state.txt maintains the current ID to regenerate the corrupted file - current_state_id=$(( $(echo "$local_minute_file" | sed 's/[^0-9]//g' | sed 's/^0*//') - 1 )) - sed -i "s/sequenceNumber=.*/sequenceNumber=$current_state_id/" "$workingDirectory/state.txt" - rm "$local_minute_file" - echo "Stopping any existing Osmosis processes..." - pkill -f "osmosis.*--replicate-apidb" - echo "Regenerating $local_minute_file..." - send_slack_message "${ENVIROMENT}: Corrupted file $local_minute_file detected. Regenerating the file..." - generate_replication - fi - fi - fi - done +# ============================================================================ +# Function: Execute osmdbt replication cycle (following official osmdbt workflow) +# ============================================================================ +# Official workflow per osmdbt documentation: +# 1. osmdbt-catchup (process old log files if any) +# 2. osmdbt-get-log (get new changes) +# 3. osmdbt-catchup (update database to new log file) +# 4. osmdbt-create-diff (create OSM change files) +# ============================================================================ +function execute_replication_cycle() { + echo "$(date +%F_%H:%M:%S): Starting replication cycle (following osmdbt official workflow)..." + + # Clean up any existing lock files + find "$workingDirectory" -name "replicate.lock" -delete + find "$runDirectory" -name "*.lock" -delete + + # Step 1: Catch up old log files (if there are complete log files left over from a crash) + echo "$(date +%F_%H:%M:%S): Step 1: Running osmdbt-catchup (processing old log files if any)..." + if ! /osmdbt/build/src/osmdbt-catchup -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-catchup.log"; then + local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-catchup (step 1) failed" + echo "$(date +%F_%H:%M:%S): $error_msg" + send_slack_message "$error_msg" + return 1 + fi + + # Step 2: Get new log file with changes + echo "$(date +%F_%H:%M:%S): Step 2: Running osmdbt-get-log (fetching new changes)..." + if ! /osmdbt/build/src/osmdbt-get-log -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-get-log.log"; then + local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-get-log failed" + echo "$(date +%F_%H:%M:%S): $error_msg" + send_slack_message "$error_msg" + return 1 + fi + + # Step 3: Catch up database to new log file + echo "$(date +%F_%H:%M:%S): Step 3: Running osmdbt-catchup (updating database to new log file)..." + if ! /osmdbt/build/src/osmdbt-catchup -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-catchup.log"; then + local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-catchup (step 3) failed" + echo "$(date +%F_%H:%M:%S): $error_msg" + send_slack_message "$error_msg" + return 1 + fi + + # Step 4: Create OSM diff files from log files + echo "$(date +%F_%H:%M:%S): Step 4: Running osmdbt-create-diff (creating OSM change files)..." + if ! /osmdbt/build/src/osmdbt-create-diff -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-create-diff.log"; then + local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-create-diff failed" + echo "$(date +%F_%H:%M:%S): $error_msg" + send_slack_message "$error_msg" + return 1 + fi + + # Find the most recent .osc.gz file + local latest_osc=$(find "$changesDir" -name "*.osc.gz" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) + + if [ -n "$latest_osc" ] && [ -f "$latest_osc" ]; then + echo "$(date +%F_%H:%M:%S): Found new OSC file: $latest_osc" + + # Check if already processed + if grep -q "$latest_osc: SUCCESS" "$processed_files_log" 2>/dev/null; then + echo "$(date +%F_%H:%M:%S): File already processed: $latest_osc" + return 0 + fi + + # Upload files to S3 + if upload_replication_files "$latest_osc"; then + echo "$(date +%F_%H:%M:%S): Replication cycle completed successfully" + verify_sequence_continuity + return 0 else - echo "File $processed_files_log not found." - echo $processed_files_log >$processed_files_log + local error_msg="🚨 ${ENVIROMENT:-production}: Failed to upload replication files for $latest_osc" + echo "$(date +%F_%H:%M:%S): $error_msg" + return 1 fi - sleep 10s + else + echo "$(date +%F_%H:%M:%S): No new OSC file generated in this cycle" + return 0 + fi +} + +# ============================================================================ +# Function: Wait for PostgreSQL to be ready +# ============================================================================ +function wait_for_postgresql() { + echo "$(date +%F_%H:%M:%S): Waiting for PostgreSQL to be ready..." + local max_attempts=30 + local attempt=0 + + while [ $attempt -lt $max_attempts ]; do + if pg_isready -h "${POSTGRES_HOST:-localhost}" -p "${POSTGRES_PORT:-5432}" >/dev/null 2>&1; then + echo "$(date +%F_%H:%M:%S): PostgreSQL is ready." + return 0 + fi + attempt=$((attempt + 1)) + echo "$(date +%F_%H:%M:%S): PostgreSQL not ready yet, attempt $attempt/$max_attempts..." + sleep 2 done + + local error_msg="🚨 ${ENVIROMENT:-production}: PostgreSQL is not ready after $max_attempts attempts" + echo "$(date +%F_%H:%M:%S): $error_msg" + return 1 } -function generate_replication() { - # Replicate the API database using Osmosis - osmosis -q \ - --replicate-apidb \ - iterations=0 \ - minInterval=60000 \ - maxInterval=120000 \ - host=$POSTGRES_HOST \ - database=$POSTGRES_DB \ - user=$POSTGRES_USER \ - password=$POSTGRES_PASSWORD \ - validateSchemaVersion=no \ - --write-replication \ - workingDirectory=$workingDirectory +# ============================================================================ +# MAIN PROCESS +# ============================================================================ +function main() { + echo "============================================================================" + echo "$(date +%F_%H:%M:%S): Starting OSM Replication Service" + echo "============================================================================" + echo "Working Directory: $workingDirectory" + echo "Changes Directory: $changesDir" + echo "Replication Slot: $REPLICATION_SLOT" + echo "Replication Interval: ${REPLICATION_INTERVAL}s" + echo "S3 Bucket: ${AWS_S3_BUCKET:-not configured}" + echo "============================================================================" + + # Initialize processed files log + touch "$processed_files_log" + + # Wait for PostgreSQL + if ! wait_for_postgresql; then + exit 1 + fi + + # Enable osmdbt replication (creates slot and sets up replication) + if ! enable_osmdbt_replication; then + exit 1 + fi + + # Recover state from S3 or local storage + recover_state_file + + # Initial cleanup + cleanup_orphaned_files + + # Main replication loop + local last_cleanup=$(date +%s) + + echo "$(date +%F_%H:%M:%S): Entering replication loop (interval: ${REPLICATION_INTERVAL}s)..." + + while true; do + local cycle_start=$(date +%s) + + # Execute replication cycle + execute_replication_cycle + + # Periodic cleanup (every CLEANUP_INTERVAL seconds) + local current_time=$(date +%s) + if [ $((current_time - last_cleanup)) -ge $CLEANUP_INTERVAL ]; then + cleanup_orphaned_files + last_cleanup=$current_time + fi + + # Calculate sleep time to maintain 1-minute interval + local cycle_duration=$(($(date +%s) - cycle_start)) + local sleep_time=$((REPLICATION_INTERVAL - cycle_duration)) + + if [ $sleep_time -gt 0 ]; then + echo "$(date +%F_%H:%M:%S): Cycle completed in ${cycle_duration}s. Sleeping for ${sleep_time}s..." + sleep $sleep_time + else + echo "$(date +%F_%H:%M:%S): WARNING: Cycle took ${cycle_duration}s (longer than ${REPLICATION_INTERVAL}s interval)" + fi + done } -######################## Start minutes replication process ######################## -get_current_state_file -flag=true -while "$flag" = true; do - pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue - flag=false - generate_replication & - monitor_minute_replication -done +main diff --git a/images/taginfo/start.sh b/images/taginfo/start.sh index 951d6a41..2c5d69c5 100755 --- a/images/taginfo/start.sh +++ b/images/taginfo/start.sh @@ -48,8 +48,8 @@ process_data() { ./update_all.sh $DATADIR mv $DATADIR/*.db $DATADIR/ mv $DATADIR/*/*.db $DATADIR/ - # if BUCKET_NAME is set upload data - if ! aws s3 ls "s3://$BUCKET_NAME/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then + # if AWS_S3_BUCKET is set upload data + if ! aws s3 ls "s3://$AWS_S3_BUCKET/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then aws s3 sync $DATADIR/ s3://$AWS_S3_BUCKET/$ENVIRONMENT/ --exclude "*" --include "*.db" fi } diff --git a/osm-seed/templates/db/db-statefulset.yaml b/osm-seed/templates/db/db-statefulset.yaml index c999445f..e55dfcd0 100644 --- a/osm-seed/templates/db/db-statefulset.yaml +++ b/osm-seed/templates/db/db-statefulset.yaml @@ -32,8 +32,8 @@ spec: containerPort: 5432 protocol: TCP env: - - name: GET_HOSTS_FROM - value: dns + # - name: GET_HOSTS_FROM + # value: dns - name: POSTGRES_HOST value: {{ .Release.Name }}-db - name: POSTGRES_DB @@ -47,7 +47,7 @@ spec: - name: ENVIRONMENT value: {{ .Values.environment }} - name: PGDATA - value: {{ .Values.db.persistenceDisk.mountPath }} + value: {{ .Values.db.env.PGDATA }} - name: POD_IP valueFrom: { fieldRef: { fieldPath: status.podIP } } {{- if .Values.db.postgresqlConfig.enabled }} @@ -76,8 +76,8 @@ spec: periodSeconds: 5 volumeMounts: - name: postgres-storage - mountPath: /var/lib/postgresql/data - subPath: {{ .Values.db.persistenceDisk.subPath }} + mountPath: {{ .Values.db.persistenceDisk.mountPath }} + # subPath: {{ .Values.db.persistenceDisk.subPath }} - name: shared-memory mountPath: /dev/shm {{- if .Values.db.postgresqlConfig.enabled }} diff --git a/osm-seed/templates/jobs/replication-job-deployment.yaml b/osm-seed/templates/jobs/replication-job-deployment.yaml index 9e0ba5bd..c3608be1 100644 --- a/osm-seed/templates/jobs/replication-job-deployment.yaml +++ b/osm-seed/templates/jobs/replication-job-deployment.yaml @@ -55,6 +55,8 @@ spec: value: {{ quote .Values.db.env.POSTGRES_PASSWORD }} - name: POSTGRES_USER value: {{ .Values.db.env.POSTGRES_USER }} + - name: POSTGRES_PORT + value: {{ .Values.db.env.POSTGRES_PORT | quote}} - name: REPLICATION_FOLDER value: replication/minute - name: CLOUDPROVIDER diff --git a/osm-seed/templates/taginfo/taginfo-cronJob.yaml b/osm-seed/templates/taginfo/taginfo-cronJob.yaml index ea06ed5b..a693c8e8 100644 --- a/osm-seed/templates/taginfo/taginfo-cronJob.yaml +++ b/osm-seed/templates/taginfo/taginfo-cronJob.yaml @@ -19,6 +19,7 @@ spec: spec: template: spec: + serviceAccountName: {{ .Values.taginfo.serviceAccount.name }} containers: - name: {{ .Release.Name }}-taginfo-job image: "{{ .Values.taginfo.image.name }}:{{ .Values.taginfo.image.tag }}" @@ -41,9 +42,19 @@ spec: cpu: {{ .Values.taginfo.cronjob.resources.limits.cpu }} {{- end }} restartPolicy: Never - {{- if .Values.taginfo.cronjob.nodeSelector.enabled }} - nodeSelector: - {{ .Values.taginfo.cronjob.nodeSelector.label_key }}: {{ .Values.taginfo.cronjob.nodeSelector.label_value }} + # Affinity settings + {{- if .Values.taginfo.cronjob.nodeAffinity.enabled }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .Values.taginfo.cronjob.nodeAffinity.key }} + operator: In + values: + {{- range .Values.taginfo.cronjob.nodeAffinity.values }} + - {{ . | quote }} + {{- end }} {{- end }} {{- end }} {{- end }} \ No newline at end of file diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index b6e76723..e51e32c8 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -1059,6 +1059,10 @@ taginfo: limits: memory: "14Gi" cpu: "3800m" + nodeAffinity: + enabled: false + key: "nodegroup_type" + values: ["job"] nodeSelector: enabled: false label_key: nodegroup_type