diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 860d1d3e9..999018a00 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,7 +1,7 @@ # full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_prewarm, pg_surgery, pg_visibility, pgstattuple # omitted because deprecated: intagg, xml2 -supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, insert_username, intarray, isn, ltree, moddatetime, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' +supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, insert_username, intarray, isn, ltree, moddatetime, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, orioledb' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 245570fa4..29f44081a 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.0.143" +postgres-version = "15.1.0.144" diff --git a/docker/orioledb/Dockerfile b/docker/orioledb/Dockerfile index 56b24831c..7b45e8c24 100644 --- a/docker/orioledb/Dockerfile +++ b/docker/orioledb/Dockerfile @@ -41,7 +41,7 @@ ARG wal_g_release=2.0.1 #################### # Install postgres #################### -FROM orioledb/orioledb:beta3-s3-pg${postgresql_major}-ubuntu as base +FROM orioledb/orioledb:latest-pg${postgresql_major}-ubuntu as base # Redeclare args for use in subsequent stages ARG TARGETARCH ARG postgresql_major @@ -77,10 +77,10 @@ ARG CACHE_EPOCH FROM builder as rust-toolchain ENV PATH=/root/.cargo/bin:$PATH RUN apt-get update && apt-get install -y --no-install-recommends curl pkg-config && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \ - rustup --version && \ - rustc --version && \ - cargo --version + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \ + rustup --version && \ + rustc --version && \ + cargo --version RUN cargo install cargo-pgrx --version 0.10.2 --locked RUN cargo pgrx init --pg${postgresql_major} $(which pg_config) @@ -321,27 +321,11 @@ RUN tar -xvf /tmp/timescaledb.tar.gz -C /tmp && \ # Build from source WORKDIR /tmp/timescaledb-${timescaledb_release}/build RUN cmake -DAPACHE_ONLY=1 .. -# error: too few arguments to function ‘table_multi_insert’ +# error: too few arguments to function ‘table_tuple_update’ +# error: too few arguments to function ‘table_tuple_delete’ RUN sed -i \ - -e "358s|^| bool *insert_indexes = NULL;|g" \ - -e "364s|);|, insert_indexes);|g" \ - -e "930s|^| bool *insert_indexes = NULL;|g" \ - -e "1115s|);|, insert_indexes);|g" \ - -e "1134s|);|, insert_indexes);|g" \ - ../src/copy.c -# error: static declaration of ‘replace_nestloop_params’ follows non-static declaration -RUN sed -i \ - -e "48s|static ||g" \ - -e "943s|static ||g" \ - ../src/import/planner.c -# error: too few arguments to function ‘table_tuple_insert’ -RUN sed -i \ - -e "1672s|*/|*/ bool *insert_indexes = NULL;|g" \ - -e "1673s|);|, insert_indexes);|g" \ - -e "1971s|*/|*/ bool *insert_indexes = NULL;|g" \ - -e "1981s|);|, insert_indexes);|g" \ - -e "2559s|}|} bool *insert_indexes = NULL;|g" \ - -e "2567s|);|, insert_indexes);|g" \ + -e "1981s|);|, NULL);|g" \ + -e "2567s|);|, NULL);|g" \ ../src/nodes/hypertable_modify.c RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccache \ make -j$(nproc) @@ -562,7 +546,7 @@ FROM rust-toolchain as pg_graphql-source ARG pg_graphql_release ARG pg_graphql_release_checksum ADD --checksum=${pg_graphql_release_checksum} \ - "https://github.com/supabase/pg_graphql/archive/refs/tags/v${pg_graphql_release}.tar.gz" \ + "https://github.com/supabase/pg_graphql/archive/refs/tags/v${pg_graphql_release}.tar.gz" \ /tmp/pg_graphql.tar.gz RUN tar -xvf /tmp/pg_graphql.tar.gz -C /tmp && \ rm -rf /tmp/pg_graphql.tar.gz @@ -642,7 +626,7 @@ FROM rust-toolchain as pg_jsonschema-source ARG pg_jsonschema_release ARG pg_jsonschema_release_checksum ADD --checksum=${pg_jsonschema_release_checksum} \ - "https://github.com/supabase/pg_jsonschema/archive/refs/tags/v${pg_jsonschema_release}.tar.gz" \ + "https://github.com/supabase/pg_jsonschema/archive/refs/tags/v${pg_jsonschema_release}.tar.gz" \ /tmp/pg_jsonschema.tar.gz RUN tar -xvf /tmp/pg_jsonschema.tar.gz -C /tmp && \ rm -rf /tmp/pg_jsonschema.tar.gz @@ -768,7 +752,7 @@ FROM rust-toolchain as wrappers-source ARG wrappers_release ARG wrappers_release_checksum ADD --checksum=${wrappers_release_checksum} \ - "https://github.com/supabase/wrappers/archive/refs/tags/v${wrappers_release}.tar.gz" \ + "https://github.com/supabase/wrappers/archive/refs/tags/v${wrappers_release}.tar.gz" \ /tmp/wrappers.tar.gz RUN tar -xvf /tmp/wrappers.tar.gz -C /tmp && \ rm -rf /tmp/wrappers.tar.gz @@ -1023,6 +1007,7 @@ COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ RUN sed -i \ -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|shared_preload_libraries = '\(.*\)'|shared_preload_libraries = '\1, orioledb'|" \ -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ @@ -1044,6 +1029,10 @@ RUN sed -i \ -e "s|PGHOST= PGHOSTADDR=|PGHOST=\$POSTGRES_HOST|g" \ /usr/local/bin/docker-entrypoint.sh +# TODO: support s3 credentials once upstream is tested +# COPY docker/orioledb/entrypoint.sh / +# ENTRYPOINT ["/entrypoint.sh"] + HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost STOPSIGNAL SIGINT EXPOSE 5432 diff --git a/docker/orioledb/entrypoint.sh b/docker/orioledb/entrypoint.sh new file mode 100755 index 000000000..fda17e766 --- /dev/null +++ b/docker/orioledb/entrypoint.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -eou pipefail + +PG_CONF=/etc/postgresql/postgresql.conf + +function configure_orioledb { + echo "Configuring OrioleDB..." + + sed -i \ + -e "s|#max_worker_processes = .*|max_worker_processes = 50 # should fit orioledb.s3_num_workers as long as other workers|" \ + -e "s|#log_min_messages = .*|log_min_messages = debug1 # will log all S3 requests|" \ + -e "s|#archive_mode = off\(.*\)|archive_mode = on\1|" \ + "$PG_CONF" + + echo " +archive_library = 'orioledb' +orioledb.main_buffers = 1GB +orioledb.undo_buffers = 256MB +orioledb.s3_num_workers = 20 # should be enough for comfortable work +orioledb.s3_mode = true +orioledb.s3_host = '$S3_HOST' # replace with your bucket URL, accelerated buckets are recommended +orioledb.s3_region = '$S3_REGION' # replace with your S3 region +orioledb.s3_accesskey = '$S3_ACCESS_KEY' # replace with your S3 key +orioledb.s3_secretkey = '$S3_SECRET_KEY' # replace with your S3 secret key +" >> "$PG_CONF" +} + +if ! grep -q orioledb "$PG_CONF"; then + configure_orioledb +fi + +docker-entrypoint.sh "$@"