diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index b570690a5a0..bac829d8a42 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -325,6 +325,10 @@ jobs: "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, + {"test":"ic-datalake-fdw", + "make_configs":["contrib/datalake_fdw:installcheck"], + "shared_preload_libraries":"datalake_fdw" + }, {"test":"ic-gpcontrib", "make_configs":["gpcontrib/orafce:installcheck", "gpcontrib/zstd:installcheck", diff --git a/contrib/Makefile b/contrib/Makefile index 3a2591e0366..c2a1d396bb8 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -25,6 +25,7 @@ SUBDIRS = \ btree_gin \ btree_gist \ citext \ + datalake_fdw \ dblink \ dict_int \ dict_xsyn \ diff --git a/contrib/datalake_fdw/.gitignore b/contrib/datalake_fdw/.gitignore new file mode 100644 index 00000000000..e769026571a --- /dev/null +++ b/contrib/datalake_fdw/.gitignore @@ -0,0 +1,8 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ + +# Generated from exports.txt at build time +/exports.map +/exports_darwin.list diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile new file mode 100644 index 00000000000..bd1d0179513 --- /dev/null +++ b/contrib/datalake_fdw/Makefile @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/Makefile + +MODULE_big = datalake_fdw +EXTENSION = datalake_fdw +DATA = datalake_fdw--1.0.sql + +OBJS = \ + src/am_iceberg/pg_iceberg_am_handler.o \ + src/am_iceberg/pg_iceberg_extensible.o \ + src/am_iceberg/pg_iceberg_ddl.o \ + src/am_iceberg/pg_iceberg_options.o \ + src/am_iceberg/pg_iceberg_guc.o \ + src/am_iceberg/pg_iceberg_reject.o \ + src/iceberg_catalog_fdw/iceberg_catalog_fdw.o \ + src/iceberg_catalog_fdw/iceberg_catalog_option.o \ + src/iceberg_volume_fdw/iceberg_volume_fdw.o \ + src/iceberg_volume_fdw/iceberg_volume_option.o \ + src/meta/meta_engine_registry.o \ + src/meta/meta_engine_init.o \ + src/meta/engine_stub/stub_engine.o \ + src/format/format_registry.o \ + src/common/dl_err.o \ + src/common/dl_option_util.o \ + src/common/parser_option.o \ + src/common/file_system_wrapper.o \ + src/common/s3_file_system.o \ + src/common/backend_registry.o + +# Use the documented PGXS knobs: pgxs.mk appends these AFTER the flags configure +# chose, so optimization/warning settings survive. A pre-include +# "override CFLAGS +=" would give CFLAGS override origin and silently discard +# Makefile.global's own "CFLAGS = @CFLAGS@" assignment. +PG_CFLAGS = -fvisibility=hidden +PG_CXXFLAGS = -fvisibility=hidden -fvisibility-inlines-hidden -std=c++17 +PG_CPPFLAGS = -I$(srcdir)/src + +# The regression cases live with the rest of the test material rather than in a +# second place of their own; pg_regress is pointed at them. REGRESS_OPTS is +# expanded after the --inputdir that Makefile.global supplies, so this wins. +# +# _PG_init refuses to run outside shared_preload_libraries, so any server used +# to test this module has to be started with it. For an in-tree "make check" +# that is what the temp-config supplies. For "make installcheck" -- which is +# what CI runs, against a cluster created with the library already preloaded -- +# pg_regress ignores it. Note that "make check" exists in-tree only; under PGXS +# pgxs.mk refuses the target outright. +REGRESS = iceberg_am_ddl iceberg_am_reject iceberg_am_acl +REGRESS_OPTS = --temp-config=$(srcdir)/datalake_fdw.conf \ + --inputdir=$(srcdir)/test/automation/sqlrepo/smoke/iceberg_am + +EXTRA_CLEAN = exports_darwin.list exports.map + +# Keep the aggregate target as make's default goal. +all: + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/datalake_fdw +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +# Everything below needs variables that Makefile.global defines (PORTNAME), and +# SHLIB_LINK additions still apply because the link recipe expands it when it +# runs. + +# Shared libraries are linked with $(CC) (see src/Makefile.shlib COMPILER), so a +# module containing C++ translation units must pull in the C++ runtime itself. +SHLIB_LINK += -lstdc++ + +# Arrow and other C++ dependencies land in this module later; the export list is +# the single place that decides what stays visible, so the mechanism ships now. +ifeq ($(PORTNAME), darwin) +EXPORT_LIST = exports_darwin.list +SHLIB_LINK += -Wl,-exported_symbols_list,exports_darwin.list + +exports_darwin.list: exports.txt + sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$$/d' -e 's/^/_/' $< > $@ +else +EXPORT_LIST = exports.map +SHLIB_LINK += -Wl,--version-script=exports.map -Wl,--exclude-libs,ALL + +exports.map: exports.txt + { echo '{ global:'; sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$$/d' -e 's/$$/;/' $<; echo 'local: *; };'; } > $@ +endif + +all: $(EXPORT_LIST) +$(shlib): $(EXPORT_LIST) diff --git a/contrib/datalake_fdw/datalake_fdw--1.0.sql b/contrib/datalake_fdw/datalake_fdw--1.0.sql new file mode 100644 index 00000000000..f8e87df5477 --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw--1.0.sql @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * contrib/datalake_fdw/datalake_fdw--1.0.sql + */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION datalake_fdw" to load this file. \quit + +CREATE FUNCTION iceberg_am_handler(internal) +RETURNS table_am_handler AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER iceberg_am_handler; + +CREATE FUNCTION iceberg_catalog_fdw_validator(text[], oid) +RETURNS void AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +CREATE FOREIGN DATA WRAPPER iceberg_catalog_fdw + VALIDATOR iceberg_catalog_fdw_validator; + +CREATE FUNCTION iceberg_volume_fdw_validator(text[], oid) +RETURNS void AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +CREATE FOREIGN DATA WRAPPER iceberg_volume_fdw + VALIDATOR iceberg_volume_fdw_validator; diff --git a/contrib/datalake_fdw/datalake_fdw.conf b/contrib/datalake_fdw/datalake_fdw.conf new file mode 100644 index 00000000000..7e1c7c5a785 --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw.conf @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/datalake_fdw.conf +# +# Configuration for the temporary server that "make check" starts. The module +# installs process-wide hooks, so _PG_init refuses to run outside +# shared_preload_libraries; without this the first statement that reaches the +# access method would fail to load the library instead of testing it. + +shared_preload_libraries = 'datalake_fdw' diff --git a/contrib/datalake_fdw/datalake_fdw.control b/contrib/datalake_fdw/datalake_fdw.control new file mode 100644 index 00000000000..263990ba8b1 --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw.control @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/datalake_fdw.control + +comment = 'Apache Iceberg lake tables for Cloudberry (demo skeleton)' +default_version = '1.0' +module_pathname = '$libdir/datalake_fdw' +relocatable = false diff --git a/contrib/datalake_fdw/exports.txt b/contrib/datalake_fdw/exports.txt new file mode 100644 index 00000000000..0db251366df --- /dev/null +++ b/contrib/datalake_fdw/exports.txt @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/exports.txt +# +# Single source of truth for exported symbols; the linker script for each +# platform is generated from it at build time. Adding a line here is an API +# decision, so it is one a reviewer has to see. Comment lines are stripped +# when the script is generated. + +_PG_init +Pg_magic_func +pg_finfo_iceberg_am_handler +iceberg_am_handler +pg_finfo_iceberg_catalog_fdw_validator +iceberg_catalog_fdw_validator +pg_finfo_iceberg_volume_fdw_validator +iceberg_volume_fdw_validator diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c new file mode 100644 index 00000000000..4a610fc3238 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c @@ -0,0 +1,509 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_am_handler.c + * Table access method callbacks for Iceberg tables. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/multixact.h" +#include "access/tableam.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "fmgr.h" + +PG_FUNCTION_INFO_V1(iceberg_am_handler); + +static const TupleTableSlotOps * +pg_iceberg_slot_callbacks(Relation rel pg_attribute_unused()) +{ + return &TTSOpsVirtual; +} + +static TableScanDesc +pg_iceberg_scan_begin(Relation rel pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + int nkeys pg_attribute_unused(), + struct ScanKeyData *key pg_attribute_unused(), + ParallelTableScanDesc pscan pg_attribute_unused(), + uint32 flags pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static void +pg_iceberg_scan_end(TableScanDesc scan pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static void +pg_iceberg_scan_rescan(TableScanDesc scan pg_attribute_unused(), + struct ScanKeyData *key pg_attribute_unused(), + bool set_params pg_attribute_unused(), + bool allow_strat pg_attribute_unused(), + bool allow_sync pg_attribute_unused(), + bool allow_pagemode pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static bool +pg_iceberg_scan_getnextslot(TableScanDesc scan pg_attribute_unused(), + ScanDirection direction pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static Size +pg_iceberg_parallelscan_estimate(Relation rel pg_attribute_unused()) +{ + pg_iceberg_not_supported("parallel scan"); +} + +static Size +pg_iceberg_parallelscan_initialize(Relation rel pg_attribute_unused(), + ParallelTableScanDesc pscan pg_attribute_unused()) +{ + pg_iceberg_not_supported("parallel scan"); +} + +static void +pg_iceberg_parallelscan_reinitialize(Relation rel pg_attribute_unused(), + ParallelTableScanDesc pscan pg_attribute_unused()) +{ + pg_iceberg_not_supported("parallel scan"); +} + +static struct IndexFetchTableData * +pg_iceberg_index_fetch_begin(Relation rel pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static void +pg_iceberg_index_fetch_reset(struct IndexFetchTableData *data pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static void +pg_iceberg_index_fetch_end(struct IndexFetchTableData *data pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static bool +pg_iceberg_index_fetch_tuple(struct IndexFetchTableData *scan pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + bool *call_again pg_attribute_unused(), + bool *all_dead pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static bool +pg_iceberg_tuple_fetch_row_version(Relation rel pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static bool +pg_iceberg_tuple_tid_valid(TableScanDesc scan pg_attribute_unused(), + ItemPointer tid pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static void +pg_iceberg_tuple_get_latest_tid(TableScanDesc scan pg_attribute_unused(), + ItemPointer tid pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static bool +pg_iceberg_tuple_satisfies_snapshot(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static TransactionId +pg_iceberg_index_delete_tuples(Relation rel pg_attribute_unused(), + TM_IndexDeleteOp *delstate pg_attribute_unused()) +{ + pg_iceberg_not_supported("index maintenance"); +} + +static void +pg_iceberg_tuple_insert(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + int options pg_attribute_unused(), + struct BulkInsertStateData *bistate pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT"); +} + +static void +pg_iceberg_tuple_insert_speculative(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + int options pg_attribute_unused(), + struct BulkInsertStateData *bistate pg_attribute_unused(), + uint32 specToken pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT ... ON CONFLICT"); +} + +static void +pg_iceberg_tuple_complete_speculative(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + uint32 specToken pg_attribute_unused(), + bool succeeded pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT ... ON CONFLICT"); +} + +static void +pg_iceberg_multi_insert(Relation rel pg_attribute_unused(), + TupleTableSlot **slots pg_attribute_unused(), + int nslots pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + int options pg_attribute_unused(), + struct BulkInsertStateData *bistate pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT"); +} + +static TM_Result +pg_iceberg_tuple_delete(Relation rel pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + Snapshot crosscheck pg_attribute_unused(), + bool wait pg_attribute_unused(), + TM_FailureData *tmfd pg_attribute_unused(), + bool changingPart pg_attribute_unused()) +{ + pg_iceberg_not_supported("DELETE"); +} + +static TM_Result +pg_iceberg_tuple_update(Relation rel pg_attribute_unused(), + ItemPointer otid pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + Snapshot crosscheck pg_attribute_unused(), + bool wait pg_attribute_unused(), + TM_FailureData *tmfd pg_attribute_unused(), + LockTupleMode *lockmode pg_attribute_unused(), + TU_UpdateIndexes *update_indexes pg_attribute_unused()) +{ + pg_iceberg_not_supported("UPDATE"); +} + +static TM_Result +pg_iceberg_tuple_lock(Relation rel pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + LockTupleMode mode pg_attribute_unused(), + LockWaitPolicy wait_policy pg_attribute_unused(), + uint8 flags pg_attribute_unused(), + TM_FailureData *tmfd pg_attribute_unused()) +{ + pg_iceberg_not_supported("row locking (SELECT ... FOR UPDATE)"); +} + +static void +pg_iceberg_relation_set_new_filelocator(Relation rel pg_attribute_unused(), + const RelFileLocator *newrlocator pg_attribute_unused(), + char persistence pg_attribute_unused(), + TransactionId *freezeXid, + MultiXactId *minmulti) +{ + /* + * Iceberg data lives in external object storage, so creating local smgr + * storage here would be both unnecessary and misleading. + */ + *freezeXid = InvalidTransactionId; + *minmulti = InvalidMultiXactId; + return; +} + +static void +pg_iceberg_relation_nontransactional_truncate(Relation rel pg_attribute_unused()) +{ + pg_iceberg_not_supported("TRUNCATE"); +} + +static void +pg_iceberg_relation_copy_data(Relation rel pg_attribute_unused(), + const RelFileLocator *newrlocator pg_attribute_unused()) +{ + pg_iceberg_not_supported("ALTER TABLE ... SET TABLESPACE"); +} + +static void +pg_iceberg_relation_copy_for_cluster(Relation OldTable pg_attribute_unused(), + Relation NewTable pg_attribute_unused(), + Relation OldIndex pg_attribute_unused(), + bool use_sort pg_attribute_unused(), + TransactionId OldestXmin pg_attribute_unused(), + TransactionId *xid_cutoff pg_attribute_unused(), + MultiXactId *multi_cutoff pg_attribute_unused(), + double *num_tuples pg_attribute_unused(), + double *tups_vacuumed pg_attribute_unused(), + double *tups_recently_dead pg_attribute_unused()) +{ + pg_iceberg_not_supported("CLUSTER / VACUUM FULL"); +} + +static void +pg_iceberg_relation_vacuum(Relation rel pg_attribute_unused(), + struct VacuumParams *params pg_attribute_unused(), + BufferAccessStrategy bstrategy pg_attribute_unused()) +{ + /* + * A database-wide VACUUM or autovacuum must not fail merely because it + * encounters an Iceberg table. There is no local storage to vacuum. + */ + return; +} + +static bool +pg_iceberg_scan_analyze_next_block(TableScanDesc scan pg_attribute_unused(), + BlockNumber blockno pg_attribute_unused(), + BufferAccessStrategy bstrategy pg_attribute_unused()) +{ + /* Defensive fallback; relation_acquire_sample_rows bypasses this path. */ + return false; +} + +static bool +pg_iceberg_scan_analyze_next_tuple(TableScanDesc scan pg_attribute_unused(), + TransactionId OldestXmin pg_attribute_unused(), + double *liverows pg_attribute_unused(), + double *deadrows pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + /* Defensive fallback; relation_acquire_sample_rows bypasses this path. */ + return false; +} + +static int +pg_iceberg_relation_acquire_sample_rows(Relation onerel pg_attribute_unused(), + int elevel pg_attribute_unused(), + HeapTuple *rows pg_attribute_unused(), + int targrows pg_attribute_unused(), + double *totalrows, + double *totaldeadrows) +{ + /* + * analyze.c uses this callback directly when present and therefore never + * starts a table_beginscan_analyze() scan. This lets ANALYZE succeed as a + * zero-sample no-op while ordinary scans remain unsupported. + */ + *totalrows = 0; + *totaldeadrows = 0; + return 0; +} + +static double +pg_iceberg_index_build_range_scan(Relation table_rel pg_attribute_unused(), + Relation index_rel pg_attribute_unused(), + struct IndexInfo *index_info pg_attribute_unused(), + bool allow_sync pg_attribute_unused(), + bool anyvisible pg_attribute_unused(), + bool progress pg_attribute_unused(), + BlockNumber start_blockno pg_attribute_unused(), + BlockNumber numblocks pg_attribute_unused(), + IndexBuildCallback callback pg_attribute_unused(), + void *callback_state pg_attribute_unused(), + TableScanDesc scan pg_attribute_unused()) +{ + pg_iceberg_not_supported("CREATE INDEX"); +} + +static void +pg_iceberg_index_validate_scan(Relation table_rel pg_attribute_unused(), + Relation index_rel pg_attribute_unused(), + struct IndexInfo *index_info pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + struct ValidateIndexState *state pg_attribute_unused()) +{ + pg_iceberg_not_supported("CREATE INDEX"); +} + +static uint64 +pg_iceberg_relation_size(Relation rel pg_attribute_unused(), + ForkNumber forkNumber pg_attribute_unused()) +{ + return 0; +} + +static BlockSequence * +pg_iceberg_relation_get_block_sequences(Relation rel pg_attribute_unused(), + int *numSequences) +{ + *numSequences = 0; + return palloc0(sizeof(BlockSequence)); +} + +static void +pg_iceberg_relation_get_block_sequence(Relation rel pg_attribute_unused(), + BlockNumber blkNum pg_attribute_unused(), + BlockSequence *sequence pg_attribute_unused()) +{ + pg_iceberg_not_supported("block sequence access"); +} + +static bool +pg_iceberg_relation_needs_toast_table(Relation rel pg_attribute_unused()) +{ + return false; +} + +static void +pg_iceberg_relation_estimate_size(Relation rel pg_attribute_unused(), + int32 *attr_widths pg_attribute_unused(), + BlockNumber *pages, + double *tuples, + double *allvisfrac) +{ + *pages = 0; + *tuples = 0; + *allvisfrac = 0; +} + +static bool +pg_iceberg_scan_sample_next_block(TableScanDesc scan pg_attribute_unused(), + struct SampleScanState *scanstate pg_attribute_unused()) +{ + pg_iceberg_not_supported("TABLESAMPLE"); +} + +static bool +pg_iceberg_scan_sample_next_tuple(TableScanDesc scan pg_attribute_unused(), + struct SampleScanState *scanstate pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + pg_iceberg_not_supported("TABLESAMPLE"); +} + +/* + * Optional callbacks below remain NULL. Iceberg currently has no local tuple + * scanning, index, bulk-insert, TOAST, bitmap-scan, DML-state, file-swap, or + * column-encoding implementation. + */ +static const TableAmRoutine pg_iceberg_methods = { + .type = T_TableAmRoutine, + + .slot_callbacks = pg_iceberg_slot_callbacks, + + .scan_begin = pg_iceberg_scan_begin, + .scan_begin_extractcolumns = NULL, + .scan_begin_extractcolumns_bm = NULL, + .scan_end = pg_iceberg_scan_end, + .scan_rescan = pg_iceberg_scan_rescan, + .scan_getnextslot = pg_iceberg_scan_getnextslot, + .scan_set_tidrange = NULL, + .scan_getnextslot_tidrange = NULL, + .scan_flags = NULL, + + .parallelscan_estimate = pg_iceberg_parallelscan_estimate, + .parallelscan_initialize = pg_iceberg_parallelscan_initialize, + .parallelscan_reinitialize = pg_iceberg_parallelscan_reinitialize, + + .index_fetch_begin = pg_iceberg_index_fetch_begin, + .index_fetch_reset = pg_iceberg_index_fetch_reset, + .index_fetch_end = pg_iceberg_index_fetch_end, + .index_fetch_tuple = pg_iceberg_index_fetch_tuple, + .index_unique_check = NULL, + + .tuple_fetch_row_version = pg_iceberg_tuple_fetch_row_version, + .tuple_tid_valid = pg_iceberg_tuple_tid_valid, + .tuple_get_latest_tid = pg_iceberg_tuple_get_latest_tid, + .tuple_satisfies_snapshot = pg_iceberg_tuple_satisfies_snapshot, + .index_delete_tuples = pg_iceberg_index_delete_tuples, + + .tuple_insert = pg_iceberg_tuple_insert, + .tuple_insert_speculative = pg_iceberg_tuple_insert_speculative, + .tuple_complete_speculative = pg_iceberg_tuple_complete_speculative, + .multi_insert = pg_iceberg_multi_insert, + .tuple_delete = pg_iceberg_tuple_delete, + .tuple_update = pg_iceberg_tuple_update, + .tuple_lock = pg_iceberg_tuple_lock, + .finish_bulk_insert = NULL, + + .relation_set_new_filelocator = pg_iceberg_relation_set_new_filelocator, + .relation_nontransactional_truncate = pg_iceberg_relation_nontransactional_truncate, + .relation_copy_data = pg_iceberg_relation_copy_data, + .relation_copy_for_cluster = pg_iceberg_relation_copy_for_cluster, + .relation_vacuum = pg_iceberg_relation_vacuum, + .scan_analyze_next_block = pg_iceberg_scan_analyze_next_block, + .scan_analyze_next_tuple = pg_iceberg_scan_analyze_next_tuple, + .relation_acquire_sample_rows = pg_iceberg_relation_acquire_sample_rows, + .index_build_range_scan = pg_iceberg_index_build_range_scan, + .index_validate_scan = pg_iceberg_index_validate_scan, + + .relation_size = pg_iceberg_relation_size, + .relation_get_block_sequences = pg_iceberg_relation_get_block_sequences, + .relation_get_block_sequence = pg_iceberg_relation_get_block_sequence, + .relation_needs_toast_table = pg_iceberg_relation_needs_toast_table, + .relation_toast_am = NULL, + .relation_fetch_toast_slice = NULL, + + .relation_estimate_size = pg_iceberg_relation_estimate_size, + + .scan_bitmap_next_block = NULL, + .scan_bitmap_next_tuple = NULL, + .scan_sample_next_block = pg_iceberg_scan_sample_next_block, + .scan_sample_next_tuple = pg_iceberg_scan_sample_next_tuple, + + .dml_init = NULL, + .dml_fini = NULL, + .amoptions = pg_iceberg_amoptions, + .swap_relation_files = NULL, + .validate_column_encoding_clauses = NULL, + .transform_column_encoding_clauses = NULL, +}; + +Datum +iceberg_am_handler(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(&pg_iceberg_methods); +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c new file mode 100644 index 00000000000..ee70aa7fa83 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c @@ -0,0 +1,302 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_ddl.c + * Object-access integration for the Iceberg table lifecycle. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/relation.h" +#include "am_iceberg/pg_iceberg_ddl.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "catalog/pg_class.h" +#include "cdb/cdbvars.h" +#include "commands/defrem.h" +#include "meta/iceberg_meta_engine.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/syscache.h" + +object_access_hook_type pg_iceberg_prev_object_access_hook; + +static Oid get_rel_relam(Oid relid); +static MetaCtx table_info_meta_ctx(const IcebergTableInfo *info); +static void iceberg_post_create(Oid objectId); +static void iceberg_drop(Oid objectId); + +/* + * This tree has no lsyscache get_rel_relam() helper, so provide the same + * missing-ok syscache lookup locally. + */ +static Oid +get_rel_relam(Oid relid) +{ + HeapTuple tuple; + Oid relam = InvalidOid; + + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (HeapTupleIsValid(tuple)) + { + relam = ((Form_pg_class) GETSTRUCT(tuple))->relam; + ReleaseSysCache(tuple); + } + + return relam; +} + +static MetaCtx +table_info_meta_ctx(const IcebergTableInfo *info) +{ + MetaCtx ctx = { + .catalog_name = info->catalog_name, + .namespace_name = info->opts->namespace, + .table_name = info->opts->table, + .catalog_props = info->catalog_props, + .n_catalog_props = info->n_catalog_props, + .credential_props = NULL, + .n_credential_props = 0 + }; + + return ctx; +} + +static void +iceberg_post_create(Oid objectId) +{ + Relation rel; + Oid amoid; + IcebergTableInfo *info; + + amoid = pg_iceberg_am_oid(); + if (!OidIsValid(amoid)) + return; + + rel = relation_open(objectId, AccessShareLock); + if (rel->rd_rel->relam != amoid) + { + relation_close(rel, AccessShareLock); + return; + } + + info = pg_iceberg_get_table_info_rel(rel); + InsertLakeTableEntry(objectId, info); + + if (Gp_role == GP_ROLE_DISPATCH) + { + const IcebergMetaEngine *engine = get_meta_engine(); + MetaCtx ctx = table_info_meta_ctx(info); + MetaTableDef def = {.schema_json = ""}; + MetaTable *table_metadata = NULL; + DlErrCode rc; + + /* Always cross the metadata-engine boundary through its wrapper. */ + rc = meta_engine_create_table(engine, &ctx, &def, &table_metadata); + relation_close(rel, AccessShareLock); + + /* ctx borrows from info, so release only once the call has returned. */ + pg_iceberg_free_table_info(info); + + if (rc != DL_OK) + dl_error_report(ERROR, rc, "create_table"); + return; + } + + relation_close(rel, AccessShareLock); + pg_iceberg_free_table_info(info); +} + +static void +iceberg_drop(Oid objectId) +{ + Relation rel; + Oid amoid; + IcebergTableInfo *info; + const IcebergMetaEngine *engine; + MetaCtx ctx; + DlErrCode rc; + + amoid = pg_iceberg_am_oid(); + if (!OidIsValid(amoid) || get_rel_relam(objectId) != amoid) + return; + + /* + * The dispatcher makes the single remote call; everyone else only drops + * local catalog rows and the dependencies recorded at creation. + * + * A utility-mode backend deliberately falls in the second group. It talks + * to one node, so letting it delete the remote table would remove metadata + * the other nodes still reference. The utility guard refuses the + * statements that reach a lake table directly; an indirect cascade that + * slips past it drops the local rows only, which is recoverable, unlike a + * remote catalog entry deleted on one node's say-so. + */ + if (Gp_role != GP_ROLE_DISPATCH) + return; + + /* + * Defensive: no path the regression suite covers -- DROP TABLE, DROP SERVER + * CASCADE, DROP SCHEMA CASCADE -- reaches this hook with the relation no + * longer openable, so the open below has always succeeded. It stays a try + * rather than an open because a hook that raised here would make the object + * undroppable, and there is nothing to reconstruct the mapping from anyway. + */ + rel = try_relation_open(objectId, AccessShareLock, false); + if (rel == NULL) + return; + + /* + * Resolving the mapping can fail -- a server option that no longer parses, + * a wrapper renamed out from under the table -- and DROP is exactly the + * statement that must still work in that state. Report what could not be + * cleaned up remotely and let the local drop proceed, rather than leaving + * the user with a table that cannot be dropped at all. + */ + info = NULL; + PG_TRY(); + { + info = pg_iceberg_get_table_info_rel(rel); + } + PG_CATCH(); + { + MemoryContext ctxt = MemoryContextSwitchTo(TopTransactionContext); + ErrorData *edata = CopyErrorData(); + + MemoryContextSwitchTo(ctxt); + + /* + * Only a mapping that no longer describes anything usable may be + * downgraded here. A cancelled query or an out-of-memory failure has + * nothing to do with the mapping, and turning one of those into a + * warning would drop the table while pretending the statement + * succeeded. + */ + if (edata->sqlerrcode != ERRCODE_INVALID_TABLE_DEFINITION && + edata->sqlerrcode != ERRCODE_INVALID_PARAMETER_VALUE && + edata->sqlerrcode != ERRCODE_UNDEFINED_OBJECT) + { + FreeErrorData(edata); + relation_close(rel, AccessShareLock); + PG_RE_THROW(); + } + + FlushErrorState(); + relation_close(rel, AccessShareLock); + ereport(WARNING, + (errmsg("iceberg: dropping \"%s\" without notifying the metadata engine", + get_rel_name(objectId)), + errdetail("%s", edata->message))); + FreeErrorData(edata); + return; + } + PG_END_TRY(); + + engine = get_meta_engine(); + ctx = table_info_meta_ctx(info); + + /* + * Dropping the table drops this database's reference to it. Whether the + * lake data goes too is the table's own decision, recorded in its options + * when it was created; the default is to leave it, so that dropping a + * reference cannot destroy data another reader still expects to find. + * + * A table's identity is its (catalog, namespace, name) triple, so there is + * nothing to fence this call against: a remote table under that name is by + * definition the table being dropped. + */ + rc = meta_engine_drop_table(engine, &ctx, info->opts->purge_on_drop); + relation_close(rel, AccessShareLock); + + /* ctx borrows from info, so release only once the call has returned. */ + pg_iceberg_free_table_info(info); + + /* + * Do not strand the local table when the remote drop fails. The catalog + * DROP continues and the failure stays visible; reconciling what the + * remote catalog still holds is the metadata agent's job, not something + * this skeleton can record durably. + */ + if (rc != DL_OK) + dl_error_report(WARNING, rc, "drop_table"); +} + +void +pg_iceberg_object_access(ObjectAccessType access, + Oid classId, + Oid objectId, + int subId, + void *arg) +{ + if (pg_iceberg_prev_object_access_hook) + (*pg_iceberg_prev_object_access_hook) (access, classId, objectId, + subId, arg); + + if (classId != RelationRelationId || subId != 0) + return; + + switch (access) + { + case OAT_POST_CREATE: + { + ObjectAccessPostCreate *created = (ObjectAccessPostCreate *) arg; + + /* + * Relations the system builds for itself -- the transient heap + * of a rewrite above all -- must never reach the metadata + * engine. They carry a generated name, they live only until + * the rewrite swaps them in, and creating them remotely leaves + * an orphan the moment the local work rolls back. The utility + * hook refuses the statements that rewrite a lake table; this + * is the backstop for whatever it does not see. + */ + if (created != NULL && created->is_internal) + return; + + iceberg_post_create(objectId); + } + break; + case OAT_DROP: + iceberg_drop(objectId); + break; + case OAT_TRUNCATE: + + /* + * TRUNCATE of a table created in an earlier transaction never + * reaches the access method's nontransactional path: it goes + * through relation_set_new_filelocator, which succeeds because a + * lake table has no local storage to reset. Without this event + * the statement would report success while every Iceberg data file + * stayed exactly where it was. + */ + if (OidIsValid(pg_iceberg_am_oid()) && + get_rel_relam(objectId) == pg_iceberg_am_oid()) + pg_iceberg_not_supported("TRUNCATE"); + break; + default: + break; + } +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h new file mode 100644 index 00000000000..c6cd354a732 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h @@ -0,0 +1,42 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_ddl.h + * Object-access integration for the Iceberg table lifecycle. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_DDL_H +#define PG_ICEBERG_DDL_H + +#include "catalog/objectaccess.h" + +extern object_access_hook_type pg_iceberg_prev_object_access_hook; + +extern void pg_iceberg_object_access(ObjectAccessType access, + Oid classId, + Oid objectId, + int subId, + void *arg); + +#endif /* PG_ICEBERG_DDL_H */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c new file mode 100644 index 00000000000..f675a1ef24d --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c @@ -0,0 +1,964 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_extensible.c + * Extension initialization and the Iceberg utility guard. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/genam.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "access/tableam.h" +#include "am_iceberg/pg_iceberg_ddl.h" +#include "am_iceberg/pg_iceberg_guc.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "catalog/namespace.h" +#include "catalog/pg_class.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_namespace.h" +#include "cdb/cdbvars.h" +#include "commands/defrem.h" +#include "common/backend_registry.h" +#include "fmgr.h" +#include "foreign/foreign.h" +#include "meta/iceberg_meta_engine.h" +#include "meta/meta_engine_init.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "storage/lmgr.h" +#include "tcop/utility.h" +#include "utils/fmgroids.h" +#include "utils/syscache.h" + +PG_MODULE_MAGIC; + +static ProcessUtility_hook_type prev_ProcessUtility_hook; + +static bool iceberg_is_effective_am(const char *accessMethod); +static Oid get_rel_relam(Oid relid); +static bool relid_is_iceberg(Oid relid); +static bool server_referenced_by_iceberg(Oid srvid); +static bool rangevar_is_iceberg(RangeVar *relation); +static const char *string_object_name(Node *object); +static Oid lock_object_by_name(Oid classid, + Oid (*lookup) (const char *name, bool missing_ok), + const char *name, LOCKMODE lockmode); +static bool locked_server_referenced_by_iceberg(const char *servername); +static bool utility_drop_targets_iceberg(DropStmt *stmt); +static bool alter_table_targets_iceberg_am(AlterTableStmt *stmt); +static bool alter_table_is_owner_only(AlterTableStmt *stmt); +static bool database_has_iceberg_table(void); +static bool schema_has_iceberg_table(const char *schemaname); +static bool server_belongs_to_module(const char *servername); +static bool is_module_fdw_name(const char *fdwname); +static const char *find_reloption(List *options, const char *name); +static void lock_create_servers(Oid catalog_srvid, Oid volume_srvid, + LOCKMODE lockmode); +static void unlock_create_servers(Oid catalog_srvid, Oid volume_srvid, + LOCKMODE lockmode); +static void validate_create_binding(const char *catalog_name, + const char *volume_name); +static void prepare_iceberg_create(CreateStmt *stmt); +static void reject_utility_mode_ddl(const char *subject) pg_attribute_noreturn(); +static void reject_targeted_operation(const char *operation); +static void pg_iceberg_ProcessUtility(PlannedStmt *pstmt, + const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, + QueryCompletion *qc); + +/* + * Resolve an omitted access method exactly as core CREATE TABLE does. Keep + * this as the single source of truth for every relation-creating node handled + * by the hook. + */ +static bool +iceberg_is_effective_am(const char *accessMethod) +{ + if (accessMethod != NULL) + return strcmp(accessMethod, "iceberg") == 0; + return default_table_access_method != NULL && + strcmp(default_table_access_method, "iceberg") == 0; +} + +/* + * This tree has no lsyscache get_rel_relam() helper, so provide the same + * missing-ok syscache lookup locally. + */ +static Oid +get_rel_relam(Oid relid) +{ + HeapTuple tuple; + Oid relam = InvalidOid; + + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (HeapTupleIsValid(tuple)) + { + relam = ((Form_pg_class) GETSTRUCT(tuple))->relam; + ReleaseSysCache(tuple); + } + + return relam; +} + +static bool +relid_is_iceberg(Oid relid) +{ + Oid iceberg_am_oid; + + /* + * Look the OID up every time instead of caching it: DROP EXTENSION + * followed by CREATE EXTENSION hands out a new OID, and a cached one would + * make these guards silently stop matching. The lookup is syscache-backed, + * and it must be missing-ok because the predicate is consulted for + * arbitrary relations before the extension exists. + */ + iceberg_am_oid = get_table_am_oid("iceberg", true); + + return OidIsValid(iceberg_am_oid) && OidIsValid(relid) && + get_rel_relam(relid) == iceberg_am_oid; +} + +static bool +server_referenced_by_iceberg(Oid srvid) +{ + Relation depend_rel; + ScanKeyData keys[2]; + SysScanDesc scan; + HeapTuple tuple; + bool referenced = false; + + if (!OidIsValid(srvid)) + return false; + + depend_rel = table_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&keys[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(ForeignServerRelationId)); + ScanKeyInit(&keys[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(srvid)); + + scan = systable_beginscan(depend_rel, DependReferenceIndexId, true, + NULL, lengthof(keys), keys); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_depend dependency = (Form_pg_depend) GETSTRUCT(tuple); + + if (dependency->classid == RelationRelationId && + relid_is_iceberg(dependency->objid)) + { + referenced = true; + break; + } + } + + systable_endscan(scan); + table_close(depend_rel, AccessShareLock); + + return referenced; +} + +static bool +rangevar_is_iceberg(RangeVar *relation) +{ + Oid relid; + + if (relation == NULL) + return false; + + relid = RangeVarGetRelid(relation, AccessShareLock, true); + return OidIsValid(relid) && relid_is_iceberg(relid); +} + +static const char * +string_object_name(Node *object) +{ + if (object != NULL && IsA(object, String)) + return strVal(object); + return NULL; +} + +/* + * Resolve a name to an object OID and lock that object exclusively, so that a + * guard can decide about it without racing the statement it is guarding. + * + * The re-resolution is what makes this correct rather than merely locked. + * Acquiring the lock can mean waiting, and the transactions waited for are free + * to rename this object away and give its name to a different one. A guard + * that skipped the recheck would then hold a lock on an object the statement no + * longer names, and would scan it instead of the object about to be changed. + * This is the lookup-lock-recheck loop PostgreSQL applies to relations for the + * same reason. + * + * Returns InvalidOid when the name resolves to nothing, holding no lock. + */ +static Oid +lock_object_by_name(Oid classid, + Oid (*lookup) (const char *name, bool missing_ok), + const char *name, LOCKMODE lockmode) +{ + for (;;) + { + Oid objectid = lookup(name, true); + + if (!OidIsValid(objectid)) + return InvalidOid; + + LockDatabaseObject(classid, objectid, 0, lockmode); + + if (lookup(name, true) == objectid) + return objectid; + + UnlockDatabaseObject(classid, objectid, 0, lockmode); + } +} + +/* + * The exclusive counterpart of the share lock CREATE takes on the servers it + * binds to. ALTER statements name one server, so ascending-OID order is + * trivial; preserve that ordering if a future statement form locks more than + * one. Lock before scanning so the dependency decision cannot race CREATE. + */ +static bool +locked_server_referenced_by_iceberg(const char *servername) +{ + Oid srvid; + + if (servername == NULL) + return false; + + srvid = lock_object_by_name(ForeignServerRelationId, + get_foreign_server_oid, servername, + AccessExclusiveLock); + if (!OidIsValid(srvid)) + return false; + + return server_referenced_by_iceberg(srvid); +} + +static bool +utility_drop_targets_iceberg(DropStmt *stmt) +{ + ListCell *lc; + + if (stmt->removeType == OBJECT_TABLE) + { + foreach(lc, stmt->objects) + { + RangeVar *relation = + makeRangeVarFromNameList((List *) lfirst(lc)); + + if (rangevar_is_iceberg(relation)) + return true; + } + } + else if (stmt->removeType == OBJECT_FOREIGN_SERVER) + { + foreach(lc, stmt->objects) + { + Node *object = (Node *) lfirst(lc); + + if (locked_server_referenced_by_iceberg( + string_object_name(object))) + return true; + } + } + + return false; +} + +/* + * Does this statement convert its target into a lake table? + */ +static bool +alter_table_targets_iceberg_am(AlterTableStmt *stmt) +{ + ListCell *lc; + + foreach(lc, stmt->cmds) + { + AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); + + if (cmd->subtype == AT_SetAccessMethod && + iceberg_is_effective_am(cmd->name)) + return true; + } + + return false; +} + +/* + * Does any lake table exist in this database? + * + * Used by the statements that name no relation at all, where there is nothing + * to match against and the only safe answer is to refuse if such a table could + * be reached. + */ +static bool +database_has_iceberg_table(void) +{ + Relation class_rel; + ScanKeyData key; + SysScanDesc scan; + HeapTuple tuple; + Oid iceberg_am_oid; + bool found = false; + + iceberg_am_oid = get_table_am_oid("iceberg", true); + if (!OidIsValid(iceberg_am_oid)) + return false; + + class_rel = table_open(RelationRelationId, AccessShareLock); + ScanKeyInit(&key, + Anum_pg_class_relam, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(iceberg_am_oid)); + scan = systable_beginscan(class_rel, InvalidOid, false, NULL, 1, &key); + if (HeapTupleIsValid(tuple = systable_getnext(scan))) + found = true; + systable_endscan(scan); + table_close(class_rel, AccessShareLock); + + return found; +} + +/* + * Does the named schema contain a lake table? + * + * A lake table's namespace is not just where it lives locally: it is the + * namespace this module reports to the metadata engine. Renaming the schema + * would therefore silently repoint the table at a different external namespace, + * leaving whatever it named before behind. The check is scoped to schemas that + * actually contain one, so renaming any other schema stays unaffected. + */ +static bool +schema_has_iceberg_table(const char *schemaname) +{ + Relation class_rel; + ScanKeyData key[2]; + SysScanDesc scan; + HeapTuple tuple; + Oid iceberg_am_oid; + Oid namespace_oid; + bool found = false; + + if (schemaname == NULL) + return false; + + iceberg_am_oid = get_table_am_oid("iceberg", true); + if (!OidIsValid(iceberg_am_oid)) + return false; + + /* + * The exclusive counterpart of the share lock CREATE takes on its target + * namespace, same as the server guard: without it a concurrent CREATE could + * add a lake table to this schema after the scan below and before the + * rename runs, and that table would then live in the renamed schema while + * the metadata engine had already been told the old name. + */ + namespace_oid = lock_object_by_name(NamespaceRelationId, + get_namespace_oid, schemaname, + AccessExclusiveLock); + if (!OidIsValid(namespace_oid)) + return false; + + class_rel = table_open(RelationRelationId, AccessShareLock); + ScanKeyInit(&key[0], + Anum_pg_class_relam, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(iceberg_am_oid)); + ScanKeyInit(&key[1], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(namespace_oid)); + scan = systable_beginscan(class_rel, InvalidOid, false, NULL, 2, key); + if (HeapTupleIsValid(tuple = systable_getnext(scan))) + found = true; + systable_endscan(scan); + table_close(class_rel, AccessShareLock); + + return found; +} + +/* + * Does this ALTER TABLE do nothing but change the owner? + * + * Every other form is refused while the access method is unfinished, but this + * one has to go through: pg_dump writes ALTER TABLE ... OWNER TO for every + * table it dumps, so refusing it means refusing to restore a dump this module + * produced. Ownership is local catalog state -- it cannot reach the external + * table or change what the mapping resolves to -- so letting it through costs + * nothing that the refusal was protecting. + */ +static bool +alter_table_is_owner_only(AlterTableStmt *stmt) +{ + ListCell *lc; + + if (stmt->cmds == NIL) + return false; + + foreach(lc, stmt->cmds) + { + AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); + + if (cmd->subtype != AT_ChangeOwner) + return false; + } + + return true; +} + +/* + * Is this a server belonging to one of this module's wrappers? + * + * Distinct from "referenced by a lake table": a server can be mutated before + * any table names it, and that is exactly the window in which the coordinator + * and a segment can be left holding different options for the same server name. + */ +static bool +server_belongs_to_module(const char *servername) +{ + ForeignServer *server; + Oid catalog_fdw; + Oid volume_fdw; + + if (servername == NULL) + return false; + + server = GetForeignServerByName(servername, true); + if (server == NULL) + return false; + + catalog_fdw = pg_iceberg_catalog_fdw_oid(true); + volume_fdw = pg_iceberg_volume_fdw_oid(true); + + return (OidIsValid(catalog_fdw) && server->fdwid == catalog_fdw) || + (OidIsValid(volume_fdw) && server->fdwid == volume_fdw); +} + +/* + * Is this one of the two wrappers this extension registers? + * + * Mappings name their servers, and a server is resolved back to its wrapper by + * the wrapper's name. Renaming one therefore breaks every mapping lookup at + * once -- including on the DROP path, which then cannot tell the metadata engine + * anything. Refused whether or not a table exists yet, because the breakage is + * in the lookup rather than in any particular table. + */ +static bool +is_module_fdw_name(const char *fdwname) +{ + return fdwname != NULL && + (strcmp(fdwname, "iceberg_catalog_fdw") == 0 || + strcmp(fdwname, "iceberg_volume_fdw") == 0); +} + +static const char * +find_reloption(List *options, const char *name) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, name) == 0) + return defGetString(def); + } + + return NULL; +} + +static void +lock_create_servers(Oid catalog_srvid, Oid volume_srvid, LOCKMODE lockmode) +{ + /* + * Ascending OID order, so that two CREATEs naming the same pair in opposite + * order cannot deadlock. ALTER-side server guards take the + * AccessExclusiveLock counterpart before scanning pg_depend. + */ + if (catalog_srvid < volume_srvid) + { + LockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); + LockDatabaseObject(ForeignServerRelationId, volume_srvid, 0, lockmode); + } + else if (volume_srvid < catalog_srvid) + { + LockDatabaseObject(ForeignServerRelationId, volume_srvid, 0, lockmode); + LockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); + } + else + LockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); +} + +static void +unlock_create_servers(Oid catalog_srvid, Oid volume_srvid, LOCKMODE lockmode) +{ + UnlockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); + if (volume_srvid != catalog_srvid) + UnlockDatabaseObject(ForeignServerRelationId, volume_srvid, 0, lockmode); +} + +static void +validate_create_binding(const char *catalog_name, const char *volume_name) +{ + ForeignServer *catalog_server; + ForeignServer *volume_server; + + /* + * Resolve, lock, and only then read what was locked -- the same + * lookup-lock-recheck the guards use, for the same reason. Two things go + * wrong without it: the name can come to denote a different server while + * this backend waits for the lock, so the checks below would describe a + * server the statement no longer names; and a concurrent ALTER SERVER that + * commits during that wait leaves any copy fetched beforehand stale, so the + * definitive read has to happen afterwards. + */ + for (;;) + { + Oid catalog_srvid = get_foreign_server_oid(catalog_name, false); + Oid volume_srvid = get_foreign_server_oid(volume_name, false); + + lock_create_servers(catalog_srvid, volume_srvid, AccessShareLock); + + if (get_foreign_server_oid(catalog_name, true) == catalog_srvid && + get_foreign_server_oid(volume_name, true) == volume_srvid) + { + catalog_server = GetForeignServer(catalog_srvid); + volume_server = GetForeignServer(volume_srvid); + break; + } + + unlock_create_servers(catalog_srvid, volume_srvid, AccessShareLock); + } + + if (catalog_server->fdwid != pg_iceberg_catalog_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg catalog server", + catalog_name))); + if (volume_server->fdwid != pg_iceberg_volume_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg volume server", + volume_name))); + + pg_iceberg_check_server_usage(catalog_server->serverid); + pg_iceberg_check_server_usage(volume_server->serverid); +} + +static void +prepare_iceberg_create(CreateStmt *stmt) +{ + DistributedBy *distributed_by; + const char *catalog_name; + const char *volume_name; + + if (stmt->accessMethod == NULL) + stmt->accessMethod = pstrdup("iceberg"); + + /* + * A lake table is distributed randomly: rows live outside PostgreSQL, so no + * local key can describe where they are. That policy is injected below. + * + * A QE receives the policy the QD injected and transformed, including the + * resolved segment count, so it accepts exactly that shape. + * + * On the dispatcher, an explicit clause is accepted only when it asks for + * what would have been injected anyway. Refusing every clause looks + * stricter but is wrong in one case that matters: pg_dump writes + * DISTRIBUTED RANDOMLY into the CREATE TABLE it emits, so refusing it means + * refusing to restore a dump this module produced. A clause naming columns + * still cannot be honoured and is refused with the syntax the user wrote. + */ + if (stmt->distributedBy != NULL) + { + if (stmt->distributedBy->ptype != POLICYTYPE_PARTITIONED) + pg_iceberg_not_supported( + stmt->distributedBy->ptype == POLICYTYPE_REPLICATED ? + "DISTRIBUTED REPLICATED" : "this distribution policy"); + if (stmt->distributedBy->keyCols != NIL) + pg_iceberg_not_supported("DISTRIBUTED BY"); + } + else if (Gp_role == GP_ROLE_EXECUTE) + pg_iceberg_not_supported("DISTRIBUTED BY"); + if (stmt->partspec != NULL || stmt->partbound != NULL) + pg_iceberg_not_supported("partitioned tables"); + if (stmt->inhRelations != NIL) + pg_iceberg_not_supported("INHERITS"); + if (stmt->ofTypename != NULL) + pg_iceberg_not_supported("typed tables (OF type)"); + if (stmt->relation->relpersistence == RELPERSISTENCE_TEMP) + pg_iceberg_not_supported("TEMP tables"); + if (stmt->relation->relpersistence == RELPERSISTENCE_UNLOGGED) + pg_iceberg_not_supported("UNLOGGED tables"); + if (stmt->oncommit != ONCOMMIT_NOOP) + pg_iceberg_not_supported("ON COMMIT"); + if (stmt->tablespacename != NULL) + pg_iceberg_not_supported("TABLESPACE"); + + if (Gp_role != GP_ROLE_EXECUTE) + { + distributed_by = makeNode(DistributedBy); + distributed_by->ptype = POLICYTYPE_PARTITIONED; + distributed_by->numsegments = -1; + distributed_by->keyCols = NIL; + stmt->distributedBy = distributed_by; + } + + catalog_name = find_reloption(stmt->options, "catalog"); + if (catalog_name == NULL) + { + if (iceberg_default_catalog == NULL || + iceberg_default_catalog[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("no catalog specified"), + errhint("Specify WITH (catalog = '...') or SET iceberg.default_catalog."))); + + stmt->options = lappend(stmt->options, + makeDefElem("catalog", + (Node *) makeString( + pstrdup(iceberg_default_catalog)), + -1)); + catalog_name = iceberg_default_catalog; + } + + volume_name = find_reloption(stmt->options, "volume"); + if (volume_name == NULL) + { + if (iceberg_default_volume == NULL || + iceberg_default_volume[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("no volume specified"), + errhint("Specify WITH (volume = '...') or SET iceberg.default_volume."))); + + stmt->options = lappend(stmt->options, + makeDefElem("volume", + (Node *) makeString( + pstrdup(iceberg_default_volume)), + -1)); + volume_name = iceberg_default_volume; + } + + /* + * This makes the QD fail before dispatch; QEs repeat the checks against + * their local catalog copies. + */ + validate_create_binding(catalog_name, volume_name); +} + +static void +reject_utility_mode_ddl(const char *subject) +{ + /* + * A utility-mode backend would create, alter, or drop only local catalog + * state, without dispatch. That would break the invariant that every node + * agrees about a lake table's mapping and about the servers it names. + */ + pg_iceberg_not_supported(psprintf("utility-mode DDL on %s", subject)); +} + +static void +reject_targeted_operation(const char *operation) +{ + if (Gp_role == GP_ROLE_UTILITY) + reject_utility_mode_ddl("iceberg tables"); + pg_iceberg_not_supported(operation); +} + +static void +pg_iceberg_ProcessUtility(PlannedStmt *pstmt, + const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, + QueryCompletion *qc) +{ + Node *parsetree = pstmt->utilityStmt; + + switch (nodeTag(parsetree)) + { + case T_CreateStmt: + { + CreateStmt *stmt = (CreateStmt *) parsetree; + + if (iceberg_is_effective_am(stmt->accessMethod)) + { + if (Gp_role == GP_ROLE_UTILITY) + reject_utility_mode_ddl("iceberg tables"); + + /* + * Every successful Iceberg CREATE is mutated. Preserve a + * protected parse tree by replacing the PlannedStmt and + * mutating only its copy. + */ + if (readOnlyTree) + { + pstmt = copyObject(pstmt); + readOnlyTree = false; + parsetree = pstmt->utilityStmt; + stmt = (CreateStmt *) parsetree; + } + prepare_iceberg_create(stmt); + } + } + break; + + case T_CreateTableAsStmt: + { + CreateTableAsStmt *stmt = (CreateTableAsStmt *) parsetree; + + if (stmt->into != NULL && + iceberg_is_effective_am(stmt->into->accessMethod)) + reject_targeted_operation( + "CREATE TABLE AS / CREATE MATERIALIZED VIEW"); + } + break; + + case T_AlterTableStmt: + { + AlterTableStmt *stmt = (AlterTableStmt *) parsetree; + + if (rangevar_is_iceberg(stmt->relation)) + { + /* + * Utility mode is refused for every form, including the one + * accepted below: core does not dispatch a utility-mode + * ALTER TABLE, so an owner change made there would land on + * the connected node alone and leave the catalogs + * disagreeing about who owns the table. + */ + if (Gp_role == GP_ROLE_UTILITY) + reject_utility_mode_ddl("iceberg tables"); + + if (!alter_table_is_owner_only(stmt)) + reject_targeted_operation( + "ALTER TABLE on iceberg tables"); + } + + /* + * Converting some other table INTO a lake table has to be + * refused here as well, and the guard above does not see it: + * the relation is still a heap when the statement arrives. + * Left alone, the rewrite would reach the metadata engine + * through the transient relation it builds -- under a + * generated name, with none of the checks CREATE TABLE makes, + * including whether the user may use the servers at all. + */ + if (alter_table_targets_iceberg_am(stmt)) + reject_targeted_operation( + "ALTER TABLE ... SET ACCESS METHOD iceberg"); + } + break; + + case T_RenameStmt: + { + RenameStmt *stmt = (RenameStmt *) parsetree; + + if ((stmt->renameType == OBJECT_TABLE || + stmt->renameType == OBJECT_COLUMN) && + rangevar_is_iceberg(stmt->relation)) + reject_targeted_operation("RENAME on iceberg tables"); + if (stmt->renameType == OBJECT_FOREIGN_SERVER && + locked_server_referenced_by_iceberg( + string_object_name(stmt->object))) + reject_targeted_operation( + "RENAME on servers referenced by iceberg tables"); + if (stmt->renameType == OBJECT_SCHEMA && + schema_has_iceberg_table(stmt->subname)) + reject_targeted_operation( + "RENAME on schemas containing iceberg tables"); + if (stmt->renameType == OBJECT_FDW && + is_module_fdw_name(string_object_name(stmt->object))) + reject_targeted_operation( + "RENAME on the iceberg foreign-data wrappers"); + } + break; + + case T_AlterObjectSchemaStmt: + { + AlterObjectSchemaStmt *stmt = + (AlterObjectSchemaStmt *) parsetree; + + if (stmt->objectType == OBJECT_TABLE && + rangevar_is_iceberg(stmt->relation)) + reject_targeted_operation( + "SET SCHEMA on iceberg tables"); + } + break; + + case T_AlterOwnerStmt: + { + AlterOwnerStmt *stmt = (AlterOwnerStmt *) parsetree; + + if (stmt->objectType == OBJECT_TABLE && + rangevar_is_iceberg(stmt->relation)) + reject_targeted_operation( + "ALTER OWNER on iceberg tables"); + if (stmt->objectType == OBJECT_FOREIGN_SERVER && + locked_server_referenced_by_iceberg( + string_object_name(stmt->object))) + reject_targeted_operation( + "ALTER OWNER on servers referenced by iceberg tables"); + } + break; + + case T_AlterForeignServerStmt: + { + AlterForeignServerStmt *stmt = + (AlterForeignServerStmt *) parsetree; + + /* + * Utility mode first, and for any server of ours rather than + * only for one a table already names. Core does not dispatch a + * utility-mode ALTER SERVER, so the options would change on the + * connected node alone; a table created afterwards would then + * resolve the same server name to different options depending on + * which node resolved it, and nothing downstream would notice. + * The window is before any dependency exists, which is precisely + * what the reference check below cannot see. + */ + if (Gp_role == GP_ROLE_UTILITY && + server_belongs_to_module(stmt->servername)) + reject_utility_mode_ddl("iceberg servers"); + + /* Both VERSION and OPTIONS forms use this parse node. */ + if (locked_server_referenced_by_iceberg(stmt->servername)) + reject_targeted_operation( + "ALTER SERVER on servers referenced by iceberg tables"); + } + break; + + case T_VacuumStmt: + /* + * Plain VACUUM and ANALYZE are no-ops for iceberg tables and stay + * allowed, but VACUUM FULL must be refused here rather than in the + * table AM: rewriting a relation first creates a transient one, + * which reaches OAT_POST_CREATE and makes the metadata engine + * create a table in the remote catalog. The subsequent + * relation_copy_for_cluster error rolls back the local catalog, + * yet the remote side would keep an orphan behind. + */ + { + VacuumStmt *stmt = (VacuumStmt *) parsetree; + ListCell *lc; + bool is_full = false; + + foreach(lc, stmt->options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "full") == 0) + is_full = defGetBoolean(def); + } + + if (is_full) + { + /* + * A database-wide VACUUM FULL names no relation, so there + * is nothing to match: refuse it outright while any lake + * table exists, rather than let it reach one and rewrite + * it. + */ + if (stmt->rels == NIL) + { + if (database_has_iceberg_table()) + reject_targeted_operation( + "VACUUM FULL while iceberg tables exist"); + } + else + { + foreach(lc, stmt->rels) + { + VacuumRelation *vrel = (VacuumRelation *) lfirst(lc); + + if (rangevar_is_iceberg(vrel->relation)) + reject_targeted_operation( + "VACUUM FULL on iceberg tables"); + } + } + } + } + break; + + case T_DropStmt: + /* + * Plain DROP TABLE is supported outside utility mode; OAT_DROP + * performs the engine call. DROP SERVER protection otherwise + * comes from the dependencies recorded on every node. + */ + if (Gp_role == GP_ROLE_UTILITY && + utility_drop_targets_iceberg((DropStmt *) parsetree)) + reject_utility_mode_ddl("iceberg tables"); + break; + + default: + break; + } + + if (prev_ProcessUtility_hook) + (*prev_ProcessUtility_hook) (pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, qc); + else + standard_ProcessUtility(pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, qc); +} + +void +_PG_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("datalake_fdw must be loaded via shared_preload_libraries"), + errhint("Add \"datalake_fdw\" to shared_preload_libraries and restart the server."))); + + pg_iceberg_define_gucs(); + pg_iceberg_register_reloptions(); + DatalakeRegisterMetaEngines(); + datalake_register_storage_backends(); + + prev_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = pg_iceberg_ProcessUtility; + + pg_iceberg_prev_object_access_hook = object_access_hook; + object_access_hook = pg_iceberg_object_access; +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c new file mode 100644 index 00000000000..71dc5c53ef1 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_guc.c + * Configuration variables for Iceberg table creation. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "am_iceberg/pg_iceberg_guc.h" +#include "utils/guc.h" + +char *iceberg_default_catalog; +char *iceberg_default_volume; + +void +pg_iceberg_define_gucs(void) +{ + /* + * Do not install check hooks for these names. The servers they name are + * not necessarily present at assignment time, and assignment happens at + * different moments on the coordinator and on the segments. CREATE TABLE + * validates the values against the catalog the executing backend sees. + */ + DefineCustomStringVariable("iceberg.default_catalog", + "Default catalog server for new Iceberg tables.", + NULL, + &iceberg_default_catalog, + "", + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomStringVariable("iceberg.default_volume", + "Default volume server for new Iceberg tables.", + NULL, + &iceberg_default_volume, + "", + PGC_USERSET, + 0, + NULL, + NULL, + NULL); +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h new file mode 100644 index 00000000000..8cd8d0a4400 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_guc.h + * Configuration variables for Iceberg table creation. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_GUC_H +#define PG_ICEBERG_GUC_H + +extern char *iceberg_default_catalog; +extern char *iceberg_default_volume; + +extern void pg_iceberg_define_gucs(void); + +#endif /* PG_ICEBERG_GUC_H */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c new file mode 100644 index 00000000000..6475546223c --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c @@ -0,0 +1,616 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_options.c + * How a lake table names its catalog and volume, and how that is read back. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/relation.h" +#include "access/reloptions.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "catalog/dependency.h" +#include "catalog/objectaddress.h" +#include "catalog/pg_class.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_user_mapping.h" +#include "commands/defrem.h" +#include "foreign/foreign.h" +#include "iceberg_catalog_fdw/iceberg_catalog_option.h" +#include "iceberg_volume_fdw/iceberg_volume_option.h" +#include "miscadmin.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + +typedef struct IcebergRelOptions +{ + int32 vl_len_; + int catalog_off; + int volume_off; + int fileformat_off; + bool purge_on_drop; +} IcebergRelOptions; + +static relopt_kind iceberg_relopt_kind; + +static char *iceberg_relopt_string(IcebergRelOptions *opts, int off); +static MetaKv *defelems_to_kvs(List *options, int *n_props); +static DlErrCode invalid_location(char **errdetail, char *detail); +static bool s3_bucket_alnum(char ch); +static bool s3_bucket_char(char ch); + +/* + * Register a private reloption kind for iceberg table access-method options. + */ +void +pg_iceberg_register_reloptions(void) +{ + if (iceberg_relopt_kind != 0) + return; + + iceberg_relopt_kind = add_reloption_kind(); + + add_string_reloption(iceberg_relopt_kind, "catalog", + "iceberg catalog foreign server name", "", NULL, + AccessExclusiveLock); + add_string_reloption(iceberg_relopt_kind, "volume", + "iceberg volume foreign server name", "", NULL, + AccessExclusiveLock); + add_string_reloption(iceberg_relopt_kind, "fileformat", + "iceberg data file format", "parquet", NULL, + AccessExclusiveLock); + + /* + * Dropping the table means dropping this database's reference to it; the + * data belongs to the lake and stays there. Deleting it as well has to be + * asked for, and the answer belongs to the table rather than to a session: + * a setting could make the same DROP destroy data or not depending on who + * typed it, and a reloption travels with the table into a dump. + */ + add_bool_reloption(iceberg_relopt_kind, "purge_on_drop", + "delete the lake data when the table is dropped", + false, AccessExclusiveLock); +} + +bytea * +pg_iceberg_amoptions(Datum reloptions, char relkind, bool validate) +{ + static const relopt_parse_elt tab[] = { + {"catalog", RELOPT_TYPE_STRING, + offsetof(IcebergRelOptions, catalog_off)}, + {"volume", RELOPT_TYPE_STRING, + offsetof(IcebergRelOptions, volume_off)}, + {"fileformat", RELOPT_TYPE_STRING, + offsetof(IcebergRelOptions, fileformat_off)}, + {"purge_on_drop", RELOPT_TYPE_BOOL, + offsetof(IcebergRelOptions, purge_on_drop)} + }; + + Assert(iceberg_relopt_kind != 0); + + /* + * The option set does not depend on relkind; the same mapping applies to + * every relation kind that can carry this access method. + */ + + /* + * Whether the named servers exist is deliberately not checked here: + * amoptions runs in relcache and utility contexts where such lookups are + * unsafe or premature. The use points check instead. + */ + return (bytea *) build_reloptions(reloptions, validate, + iceberg_relopt_kind, + sizeof(IcebergRelOptions), + tab, lengthof(tab)); +} + +/* + * rd_options belongs to the relcache. Every returned string is copied into + * the caller's current memory context; callers must never retain a pointer + * into rd_options itself. + */ +static char * +iceberg_relopt_string(IcebergRelOptions *opts, int off) +{ + if (opts == NULL || off == 0) + return NULL; + + return pstrdup(((char *) opts) + off); +} + +static MetaKv * +defelems_to_kvs(List *options, int *n_props) +{ + MetaKv *props; + ListCell *lc; + int i = 0; + int count = list_length(options); + + *n_props = count; + if (count == 0) + return NULL; + + props = palloc0(sizeof(MetaKv) * count); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + props[i].key = pstrdup(def->defname); + props[i].value = pstrdup(defGetString(def)); + i++; + } + + return props; +} + +Oid +pg_iceberg_catalog_fdw_oid(bool missing_ok) +{ + ForeignDataWrapper *fdw; + + fdw = GetForeignDataWrapperByName("iceberg_catalog_fdw", missing_ok); + return fdw == NULL ? InvalidOid : fdw->fdwid; +} + +Oid +pg_iceberg_volume_fdw_oid(bool missing_ok) +{ + ForeignDataWrapper *fdw; + + fdw = GetForeignDataWrapperByName("iceberg_volume_fdw", missing_ok); + return fdw == NULL ? InvalidOid : fdw->fdwid; +} + +/* + * OID of this extension's access method, or InvalidOid while it does not + * exist. + * + * Missing-ok because the AM is absent while CREATE EXTENSION is still + * installing it. The result is deliberately not cached: DROP EXTENSION + * followed by CREATE EXTENSION produces a new OID, and a stale cached one + * would make callers silently treat lake tables as ordinary relations. The + * lookup is syscache-backed. + */ +Oid +pg_iceberg_am_oid(void) +{ + return get_table_am_oid("iceberg", true); +} + +IcebergTableInfo * +pg_iceberg_get_table_info_rel(Relation rel) +{ + IcebergRelOptions *opts; + IcebergTableInfo *info; + IcebergCatalogOptions *catalog_options; + IcebergVolumeOptions *volume_options; + ForeignServer *catalog_server; + ForeignServer *volume_server; + char *catalog_server_name; + char *volume_server_name; + char *parse_detail = NULL; + DlErrCode parse_result; + Oid amoid; + + /* + * rd_options is only an IcebergRelOptions when this access method put it + * there. Every other access method has its own layout -- a heap's + * StdRdOptions would present fillfactor and toast_tuple_target where the + * string offsets are read below, and pstrdup() would then run off the + * allocation. The callers in this module check the access method before + * getting here, but this function is reachable from anywhere, so it cannot + * rely on that. + */ + amoid = pg_iceberg_am_oid(); + if (!OidIsValid(amoid) || rel->rd_rel->relam != amoid) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not an iceberg table", + RelationGetRelationName(rel)))); + + opts = (IcebergRelOptions *) rel->rd_options; + + catalog_server_name = iceberg_relopt_string(opts, + opts == NULL ? 0 : opts->catalog_off); + volume_server_name = iceberg_relopt_string(opts, + opts == NULL ? 0 : opts->volume_off); + + /* + * fileformat is a valid option and is persisted, but nothing consumes it + * until the format layer exists, so the result does not carry it. + */ + + if (catalog_server_name == NULL || catalog_server_name[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("iceberg table \"%s\" has no catalog binding", + RelationGetRelationName(rel)), + errhint("Specify WITH (catalog = '...', volume = '...'), or set " + "iceberg.default_catalog and iceberg.default_volume " + "before CREATE TABLE."))); + + if (volume_server_name == NULL || volume_server_name[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("iceberg table \"%s\" has no volume binding", + RelationGetRelationName(rel)), + errhint("Specify WITH (catalog = '...', volume = '...'), or set " + "iceberg.default_catalog and iceberg.default_volume " + "before CREATE TABLE."))); + + catalog_server = GetForeignServerByName(catalog_server_name, false); + if (catalog_server->fdwid != pg_iceberg_catalog_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg catalog server", + catalog_server_name))); + + volume_server = GetForeignServerByName(volume_server_name, false); + if (volume_server->fdwid != pg_iceberg_volume_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg volume server", + volume_server_name))); + + catalog_options = get_iceberg_catalog_options(catalog_server); + volume_options = get_iceberg_volume_options(volume_server); + + info = palloc0(sizeof(IcebergTableInfo)); + info->catalog_name = pstrdup(catalog_options->foreign_catalog.catalog_name); + info->catalog_server_name = catalog_server_name; + info->volume_name = pstrdup(volume_server->servername); + info->volume_server_name = volume_server_name; + + info->opts = palloc0(sizeof(IcebergTableOptions)); + + /* + * Copied rather than aliased to info->catalog_name: the two fields are + * released independently. + */ + info->opts->catalog = pstrdup(info->catalog_name); + info->opts->namespace = get_namespace_name(RelationGetNamespace(rel)); + info->opts->table = pstrdup(RelationGetRelationName(rel)); + info->opts->location = NULL; + info->opts->purge_on_drop = opts != NULL && opts->purge_on_drop; + + info->catalog_srvid = catalog_server->serverid; + info->volume_srvid = volume_server->serverid; + info->catalog_props = defelems_to_kvs(catalog_server->options, + &info->n_catalog_props); + + parse_result = pg_iceberg_parse_location(volume_options->foreign_volume.base_path, + volume_options->volume_server.endpoint, + volume_options->volume_server.region, + &info->volume_location, + &parse_detail); + if (parse_result != DL_OK) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg volume server \"%s\" has an invalid %s", + volume_server_name, + DATALAKE_ICEBERG_VOLUME_BASE_PATH), + errdetail("%s", parse_detail))); + + /* + * Credential resolution intentionally does not happen here. Stub and + * DDL-only paths must be able to describe a table with zero credentials. + */ + return info; +} + +/* + * Same result from a relation OID. + * + * Not for use from an object access hook: while a relation is being dropped its + * relcache entry may already be gone, and a hook has to decide what to do about + * that rather than error out. Those callers open the relation themselves and + * use pg_iceberg_get_table_info_rel(). + */ +IcebergTableInfo * +pg_iceberg_get_table_info(Oid relid) +{ + Relation rel; + IcebergTableInfo *info; + + rel = try_relation_open(relid, AccessShareLock, false); + if (rel == NULL) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("lake table entry not found for relation %u", relid))); + + info = pg_iceberg_get_table_info_rel(rel); + relation_close(rel, AccessShareLock); + + return info; +} + +void +pg_iceberg_free_table_info(IcebergTableInfo *info) +{ + int i; + + if (info == NULL) + return; + + if (info->catalog_name) + pfree(info->catalog_name); + if (info->catalog_server_name) + pfree(info->catalog_server_name); + if (info->volume_name) + pfree(info->volume_name); + if (info->volume_server_name) + pfree(info->volume_server_name); + + if (info->opts) + { + if (info->opts->catalog) + pfree(info->opts->catalog); + if (info->opts->namespace) + pfree(info->opts->namespace); + if (info->opts->table) + pfree(info->opts->table); + if (info->opts->location) + pfree(info->opts->location); + pfree(info->opts); + } + + /* + * The parsed location and the property array are allocated by this module + * too; a destructor that released only the names would let a statement + * resolving many tables accumulate the rest until its context is reset. + */ + if (info->volume_location.scheme) + pfree(info->volume_location.scheme); + if (info->volume_location.authority) + pfree(info->volume_location.authority); + if (info->volume_location.path_prefix) + pfree(info->volume_location.path_prefix); + if (info->volume_location.endpoint) + pfree(info->volume_location.endpoint); + if (info->volume_location.region) + pfree(info->volume_location.region); + + for (i = 0; i < info->n_catalog_props; i++) + { + if (info->catalog_props[i].key) + pfree(info->catalog_props[i].key); + if (info->catalog_props[i].value) + pfree(info->catalog_props[i].value); + } + if (info->catalog_props) + pfree(info->catalog_props); + + pfree(info); +} + +/* + * Make the mapping enforceable by recording what the table depends on. + * + * The mapping itself needs no insertion: it is part of the relation, written by + * the CREATE TABLE that produced it. What has to be added is the pair of + * dependency rows that stop either server from being dropped out from under the + * table, and that carry it along when one is dropped with CASCADE. Recorded on + * the dispatcher and on every segment, so that DROP SERVER is refused locally on + * whichever node first evaluates it. + * + * There is deliberately no RemoveLakeTableEntry() counterpart: both the + * reloptions and these rows belong to the relation, so they are removed by the + * same delete that removes it. + */ +void +InsertLakeTableEntry(Oid relid, const IcebergTableInfo *info) +{ + ObjectAddress table; + ObjectAddress server; + + ObjectAddressSet(table, RelationRelationId, relid); + + ObjectAddressSet(server, ForeignServerRelationId, info->catalog_srvid); + recordDependencyOn(&table, &server, DEPENDENCY_NORMAL); + + ObjectAddressSet(server, ForeignServerRelationId, info->volume_srvid); + recordDependencyOn(&table, &server, DEPENDENCY_NORMAL); +} + +MetaKv * +pg_iceberg_resolve_credentials(Oid serverid, Oid auth_userid, int *n_props) +{ + HeapTuple tuple; + Datum options_datum; + bool isnull; + List *options; + MetaKv *props; + + Assert(n_props != NULL); + *n_props = 0; + + /* + * GetUserMapping() cannot be used here because it ereports when neither a + * user-specific nor PUBLIC mapping exists. User mappings are optional in + * v1, so perform the same two syscache probes with missing-ok semantics. + */ + tuple = SearchSysCache2(USERMAPPINGUSERSERVER, + ObjectIdGetDatum(auth_userid), + ObjectIdGetDatum(serverid)); + if (!HeapTupleIsValid(tuple) && OidIsValid(auth_userid)) + tuple = SearchSysCache2(USERMAPPINGUSERSERVER, + ObjectIdGetDatum(InvalidOid), + ObjectIdGetDatum(serverid)); + + if (!HeapTupleIsValid(tuple)) + return NULL; + + options_datum = SysCacheGetAttr(USERMAPPINGUSERSERVER, tuple, + Anum_pg_user_mapping_umoptions, + &isnull); + if (isnull) + { + ReleaseSysCache(tuple); + return NULL; + } + + options = untransformRelOptions(options_datum); + props = defelems_to_kvs(options, n_props); + ReleaseSysCache(tuple); + + return props; +} + +void +pg_iceberg_check_server_usage(Oid serverid) +{ + AclResult aclresult; + + aclresult = object_aclcheck(ForeignServerRelationId, serverid, + GetUserId(), ACL_USAGE); + if (aclresult == ACLCHECK_NO_PRIV) + aclcheck_error(aclresult, OBJECT_FOREIGN_SERVER, + GetForeignServer(serverid)->servername); +} + +static DlErrCode +invalid_location(char **errdetail, char *detail) +{ + if (errdetail != NULL) + *errdetail = detail; + else + pfree(detail); + + return DL_ERR_INVALID_OPTION; +} + +static bool +s3_bucket_alnum(char ch) +{ + return (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9'); +} + +static bool +s3_bucket_char(char ch) +{ + return s3_bucket_alnum(ch) || ch == '.' || ch == '-'; +} + +DlErrCode +pg_iceberg_parse_location(const char *uri, const char *endpoint, + const char *region, DatalakeLocation *out, + char **errdetail) +{ + const char *scheme_end; + const char *authority_start; + const char *path_start; + Size scheme_len; + Size authority_len; + Size path_len; + bool is_s3; + Size i; + + Assert(out != NULL); + memset(out, 0, sizeof(*out)); + if (errdetail != NULL) + *errdetail = NULL; + + if (uri == NULL) + return invalid_location(errdetail, + pstrdup("location URI is null")); + + scheme_end = strstr(uri, "://"); + if (scheme_end == NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" is missing \"://\"", + uri)); + + scheme_len = scheme_end - uri; + is_s3 = scheme_len == strlen(DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3) && + strncmp(uri, DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3, scheme_len) == 0; + if (!is_s3 && + !(scheme_len == strlen(DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS) && + strncmp(uri, DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS, scheme_len) == 0)) + return invalid_location(errdetail, + psprintf("location URI \"%s\" has unsupported scheme; expected %s or %s", + uri, + DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3, + DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS)); + + if (strchr(uri, '?') != NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" must not contain a query", + uri)); + if (strchr(uri, '#') != NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" must not contain a fragment", + uri)); + + authority_start = scheme_end + 3; + path_start = strchr(authority_start, '/'); + authority_len = path_start == NULL ? + strlen(authority_start) : (Size) (path_start - authority_start); + + if (authority_len == 0) + return invalid_location(errdetail, + psprintf("location URI \"%s\" has an empty authority", + uri)); + if (memchr(authority_start, '@', authority_len) != NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" must not contain userinfo", + uri)); + + if (is_s3) + { + if (authority_len < 3 || authority_len > 63) + return invalid_location(errdetail, + psprintf("s3 bucket in location URI \"%s\" must be 3 to 63 characters", + uri)); + if (!s3_bucket_alnum(authority_start[0]) || + !s3_bucket_alnum(authority_start[authority_len - 1])) + return invalid_location(errdetail, + psprintf("s3 bucket in location URI \"%s\" must start and end with a lowercase letter or digit", + uri)); + for (i = 0; i < authority_len; i++) + { + if (!s3_bucket_char(authority_start[i])) + return invalid_location(errdetail, + psprintf("s3 bucket in location URI \"%s\" contains an invalid character", + uri)); + } + } + + path_len = path_start == NULL ? 0 : strlen(path_start); + while (path_len > 0 && path_start[path_len - 1] == '/') + path_len--; + + out->schema_version = DATALAKE_LOCATION_SCHEMA_VERSION; + out->scheme = pnstrdup(uri, scheme_len); + out->authority = pnstrdup(authority_start, authority_len); + out->path_prefix = path_len == 0 ? + pstrdup("") : pnstrdup(path_start, path_len); + out->endpoint = endpoint == NULL ? NULL : pstrdup(endpoint); + out->region = region == NULL ? NULL : pstrdup(region); + + return DL_OK; +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h new file mode 100644 index 00000000000..d081ffacdb0 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h @@ -0,0 +1,155 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_options.h + * How a lake table names its catalog and volume, and how that is read back. + * + * A relation using this access method holds no data locally: its metadata lives + * in an external Iceberg catalog and its files live on external storage. The + * mapping from the relation to the two foreign servers that describe those + * places is therefore part of the table definition, and every operation on the + * table starts by reading it. + * + * pg_iceberg_get_table_info() is that read. It is the only place that knows + * where the mapping is stored -- here, the relation's own reloptions -- so the + * storage can be reconsidered later without touching a single caller. + * + * Its signature and result types are those of the reference implementation: the + * existing implementation of this feature that this work derives from and is + * meant to replace, which keeps the same mapping in a system catalog of its own + * that an extension cannot add. The difference stops inside this function, and + * code written against either one compiles against the other. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_OPTIONS_H +#define PG_ICEBERG_OPTIONS_H + +#ifdef __cplusplus +/* + * IcebergTableOptions.namespace carries the reference implementation's field + * name, which is a keyword in C++. Keeping the name is what lets that code + * move here unchanged; the cost is that this header is C-only. A C++ layer + * that needs the mapping should be given an accessor rather than this struct. + */ +#error "pg_iceberg_options.h is C-only; see the comment above this #error" +#endif + +#include "postgres.h" + +#include "common/datalake_location.h" +#include "common/dl_err.h" +#include "meta/iceberg_meta_engine.h" +#include "utils/relcache.h" + +/* + * Reference implementation: IcebergTableOptions. + * + * Deferred, names kept for the port: autovacuum_enabled, compression, + * compression_level -- all of them write-path options. + */ +typedef struct IcebergTableOptions +{ + char *catalog; /* catalog name within the external catalog */ + char *namespace; /* namespace within the external catalog */ + char *table; /* table name within the external catalog */ + char *location; /* optional location override, NULL when the + * volume's own base path applies */ + + /* + * Whether DROP TABLE should delete the lake data as well. No counterpart + * in the reference implementation, which deletes it unconditionally; here + * the default is not to, so that dropping a reference cannot destroy data + * another reader still expects to find. + */ + bool purge_on_drop; +} IcebergTableOptions; + +/* + * Reference implementation: IcebergTableInfo. + * + * The reference implementation distinguishes a catalog object from the server + * hosting it, and likewise for volumes. An extension cannot add catalog or + * volume objects to the system catalogs, so here a server names exactly one of + * each and catalog_name/volume_name fall back to the server name. Keeping all + * four fields means a caller that reads either one still gets a usable answer. + * + * Fields below the marker have no counterpart in the reference implementation. + * They stay at the end so that the shared prefix keeps its layout. + */ +typedef struct IcebergTableInfo +{ + char *catalog_name; + char *catalog_server_name; + char *volume_name; + char *volume_server_name; + IcebergTableOptions *opts; + + /* --- extension-only, keep last --- */ + Oid catalog_srvid; /* for USAGE checks and dependency records */ + Oid volume_srvid; + DatalakeLocation volume_location; /* parsed once from the volume server */ + MetaKv *catalog_props; /* catalog server options, non-secret only */ + int n_catalog_props; +} IcebergTableInfo; + +extern void pg_iceberg_register_reloptions(void); +extern bytea *pg_iceberg_amoptions(Datum reloptions, char relkind, + bool validate); + +/* + * Reference implementation: pg_iceberg_get_table_info(). Errors out when relid + * is not a lake table with a resolvable mapping. + * + * The _rel variant is what this tree calls: reading reloptions needs the + * relation anyway, so a caller holding one should not pay for a second open. + * Callers reached from object access hooks must use it -- see the note in + * pg_iceberg_options.c. + */ +extern IcebergTableInfo *pg_iceberg_get_table_info(Oid relid); +extern IcebergTableInfo *pg_iceberg_get_table_info_rel(Relation rel); +extern void pg_iceberg_free_table_info(IcebergTableInfo *info); + +/* + * Record the dependency rows that make the mapping durable. Named after the + * reference implementation's catalog-side equivalent so that the two lifecycles + * read alike. There is no removal counterpart: both the reloptions and these + * rows belong to the relation, so the delete that removes it removes them too. + */ +extern void InsertLakeTableEntry(Oid relid, const IcebergTableInfo *info); + +/* OID of this extension's access method, InvalidOid when it does not exist. */ +extern Oid pg_iceberg_am_oid(void); + +extern MetaKv *pg_iceberg_resolve_credentials(Oid serverid, Oid auth_userid, + int *n_props); +extern void pg_iceberg_check_server_usage(Oid serverid); +extern DlErrCode pg_iceberg_parse_location(const char *uri, + const char *endpoint, + const char *region, + DatalakeLocation *out, + char **errdetail); +extern Oid pg_iceberg_catalog_fdw_oid(bool missing_ok); +extern Oid pg_iceberg_volume_fdw_oid(bool missing_ok); + +#endif /* PG_ICEBERG_OPTIONS_H */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c new file mode 100644 index 00000000000..a90be0114d6 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_reject.c + * Common rejection path for unsupported Iceberg operations. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "am_iceberg/pg_iceberg_reject.h" + +void +pg_iceberg_not_supported(const char *operation) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("iceberg: %s is not supported yet", operation))); +} + +/* ------------------------------------------------------------------------ + * Utility-statement reject matrix (placeholder for the hooks task). + * ------------------------------------------------------------------------ + */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h new file mode 100644 index 00000000000..2db11592e87 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_reject.h + * Common rejection path for unsupported Iceberg operations. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_REJECT_H +#define PG_ICEBERG_REJECT_H + +#include "c.h" + +extern void pg_iceberg_not_supported(const char *operation) pg_attribute_noreturn(); + +#endif /* PG_ICEBERG_REJECT_H */ diff --git a/contrib/datalake_fdw/src/common/backend_registry.cpp b/contrib/datalake_fdw/src/common/backend_registry.cpp new file mode 100644 index 00000000000..78fb04a5e33 --- /dev/null +++ b/contrib/datalake_fdw/src/common/backend_registry.cpp @@ -0,0 +1,122 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * backend_registry.cpp + * Registry of the storage backends, one per protocol. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/backend_registry.cpp + * + *------------------------------------------------------------------------- + */ + +#include "common/dl_pg_api.h" + +#include + +#include "common/backend_registry.h" +#include "common/dl_wrappers.h" + +typedef struct DatalakeStorageBackend +{ + const char *scheme; + const struct DatalakeStorageOps *ops; +} DatalakeStorageBackend; + +/* Room for s3 and hdfs, plus space to grow without revisiting this. */ +static DatalakeStorageBackend storage_backends[4]; +static int nstorage_backends; + +extern DlErrCode datalake_register_s3_backend(void); + +static bool +storage_ops_are_complete(const struct DatalakeStorageOps *ops) +{ + /* + * A partially filled table would turn into a null call at the first + * operation the backend forgot, so refuse it at registration instead. + */ + return ops != NULL && + ops->fs_open != NULL && + ops->fs_close != NULL && + ops->fs_list != NULL && + ops->file_open != NULL && + ops->file_read != NULL && + ops->file_write != NULL && + ops->file_close != NULL && + ops->file_abort != NULL; +} + +DlErrCode +datalake_register_storage_backend(const char *scheme, + const struct DatalakeStorageOps *ops) +{ + int i; + + if (scheme == NULL || scheme[0] == '\0' || !storage_ops_are_complete(ops)) + return DL_ERR_INVALID_OPTION; + + for (i = 0; i < nstorage_backends; i++) + { + if (strcmp(storage_backends[i].scheme, scheme) == 0) + return DL_ERR_ALREADY_EXISTS; + } + + if (nstorage_backends >= (int) lengthof(storage_backends)) + return DL_ERR_INTERNAL; + + storage_backends[nstorage_backends].scheme = scheme; + storage_backends[nstorage_backends].ops = ops; + nstorage_backends++; + + return DL_OK; +} + +const struct DatalakeStorageOps * +datalake_lookup_storage_backend(const char *scheme) +{ + int i; + + if (scheme == NULL) + return NULL; + + for (i = 0; i < nstorage_backends; i++) + { + if (strcmp(storage_backends[i].scheme, scheme) == 0) + return storage_backends[i].ops; + } + + return NULL; +} + +extern "C" void +datalake_register_storage_backends(void) +{ + DL_TRY + { + DlErrCode rc = datalake_register_s3_backend(); + + /* Registering twice is harmless; anything else is a coding error. */ + if (rc != DL_OK && rc != DL_ERR_ALREADY_EXISTS) + ereport(ERROR, + (errmsg("datalake_fdw: could not register the s3 storage backend: %s", + dl_err_message(rc)))); + } + DL_CATCH_END(); +} diff --git a/contrib/datalake_fdw/src/common/backend_registry.h b/contrib/datalake_fdw/src/common/backend_registry.h new file mode 100644 index 00000000000..5432502ed42 --- /dev/null +++ b/contrib/datalake_fdw/src/common/backend_registry.h @@ -0,0 +1,100 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * backend_registry.h + * Registry of the storage backends, one per protocol. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/backend_registry.h + * + *------------------------------------------------------------------------- + */ + +#ifndef BACKEND_REGISTRY_H +#define BACKEND_REGISTRY_H + +#include + +#include "common/file_system_wrapper.h" + +#ifdef __cplusplus + +/* + * One storage protocol's implementation of the facade in + * common/file_system_wrapper.h. The operations mirror it one for one, so a + * backend is written against the same contract its callers see. + */ +struct DatalakeStorageOps +{ + DlErrCode (*fs_open) (const DatalakeLocation *location, + const DlKeyValue *credentials, int ncredentials, + DatalakeFileSystem *fs_out); + void (*fs_close) (DatalakeFileSystem fs); /* releases fs */ + DlErrCode (*fs_list) (DatalakeFileSystem fs, const char *prefix, + char ***names_out, int *nnames_out); + DlErrCode (*file_open) (DatalakeFileSystem fs, const char *path, + DatalakeFileMode mode, DatalakeFile *file_out); + DlErrCode (*file_read) (DatalakeFile file, void *buffer, int64_t length, + int64_t *nread); + DlErrCode (*file_write) (DatalakeFile file, const void *buffer, + int64_t length); + DlErrCode (*file_close) (DatalakeFile file); /* releases file */ + void (*file_abort) (DatalakeFile file); /* releases file */ +}; + +/* + * Every handle a backend hands out starts with this field, which is how the + * facade finds its way back to the right operations. A handle lives until a + * cleanup entry point consumes it; there is no closed-but-alive state, because + * keeping one would mean either leaking every handle or letting a backend free + * memory the facade still reads. + */ +struct DatalakeFileSystemData +{ + const struct DatalakeStorageOps *ops; +}; + +struct DatalakeFileData +{ + const struct DatalakeStorageOps *ops; +}; + +extern DlErrCode datalake_register_storage_backend(const char *scheme, + const struct DatalakeStorageOps *ops); +extern const struct DatalakeStorageOps *datalake_lookup_storage_backend(const char *scheme); + +#endif /* __cplusplus */ + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* + * Registration is an explicit call rather than a static initializer: the order + * static initializers run in a shared module is not something to depend on, + * and _PG_init is where this is meant to happen. + */ +extern void datalake_register_storage_backends(void); + +#ifdef __cplusplus +} +#endif + +#endif /* BACKEND_REGISTRY_H */ diff --git a/contrib/datalake_fdw/src/common/datalake_location.h b/contrib/datalake_fdw/src/common/datalake_location.h new file mode 100644 index 00000000000..28bede6f71f --- /dev/null +++ b/contrib/datalake_fdw/src/common/datalake_location.h @@ -0,0 +1,50 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * datalake_location.h + * The canonical form of a lake table storage location. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/datalake_location.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DATALAKE_LOCATION_H +#define DATALAKE_LOCATION_H + +#include + +/* Canonical, versioned location form. URIs are parsed ONCE (options layer); + * every backend receives only this struct and must never re-parse URIs. */ +typedef struct DatalakeLocation { + uint32_t schema_version; /* = 1 */ + char *scheme; /* v1 whitelist: "s3" | "hdfs" */ + char *authority; /* s3: bucket (validated); hdfs: namenode[:port] */ + char *path_prefix; /* normalized: always starts with '/', never ends with '/' + * (a bare "/" normalizes to "") */ + char *endpoint; /* optional, may be NULL */ + char *region; /* optional, may be NULL */ +} DatalakeLocation; +#define DATALAKE_LOCATION_SCHEMA_VERSION 1 + +/* Join paths as full = path_prefix + "/" + relative; relative never starts + * with '/'. */ + +#endif /* DATALAKE_LOCATION_H */ diff --git a/contrib/datalake_fdw/src/common/dl_err.c b/contrib/datalake_fdw/src/common/dl_err.c new file mode 100644 index 00000000000..4780d9d44a0 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_err.c @@ -0,0 +1,218 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_err.c + * Error codes shared by the layers below the access method, and the + * channel that carries what the code alone cannot say. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_err.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/dl_err.h" +#include "utils/guc.h" + +/* + * One record per backend. A backend runs one statement at a time, and nothing + * here outlives the report it feeds, so there is no reason to key this by + * anything finer. + */ +static DlErrorDetail dl_error_detail; + +static void dl_error_copy_field(char *dest, Size dest_size, const char *src); +static int dl_error_sqlstate(DlErrCode code); + +/* + * SQLSTATE for a failure below the access method. + * + * Only a defect in this module is an internal error. A remote catalog that + * refuses a name, or storage that will not answer, is a condition a client can + * act on and deserves a code that says so -- and reporting everything as + * ERRCODE_INTERNAL_ERROR has a second cost here: this server appends the + * raising source location to internal errors, which puts a file and line number + * into user-visible output and into every expected-output file. + */ +static int +dl_error_sqlstate(DlErrCode code) +{ + switch (code) + { + case DL_OK: + case DL_ERR_INTERNAL: + return ERRCODE_INTERNAL_ERROR; + case DL_ERR_NOT_SUPPORTED: + return ERRCODE_FEATURE_NOT_SUPPORTED; + case DL_ERR_INVALID_OPTION: + return ERRCODE_INVALID_PARAMETER_VALUE; + case DL_ERR_NOT_FOUND: + return ERRCODE_UNDEFINED_OBJECT; + case DL_ERR_ALREADY_EXISTS: + return ERRCODE_DUPLICATE_TABLE; + case DL_ERR_IO: + return ERRCODE_IO_ERROR; + } + + return ERRCODE_INTERNAL_ERROR; +} + +static void +dl_error_copy_field(char *dest, Size dest_size, const char *src) +{ + if (src == NULL) + { + dest[0] = '\0'; + return; + } + + /* Truncates rather than failing; see the header for why. */ + strlcpy(dest, src, dest_size); +} + +void +dl_error_reset(void) +{ + dl_error_detail.code = DL_OK; + dl_error_detail.remote_code = 0; + dl_error_detail.operation[0] = '\0'; + dl_error_detail.type[0] = '\0'; + dl_error_detail.message[0] = '\0'; + dl_error_detail.stack[0] = '\0'; +} + +void +dl_error_set(DlErrCode code, const char *operation, const char *type, + const char *message) +{ + dl_error_detail.code = code; + dl_error_detail.remote_code = 0; + dl_error_copy_field(dl_error_detail.operation, + sizeof(dl_error_detail.operation), operation); + dl_error_copy_field(dl_error_detail.type, + sizeof(dl_error_detail.type), type); + dl_error_copy_field(dl_error_detail.message, + sizeof(dl_error_detail.message), message); + dl_error_detail.stack[0] = '\0'; +} + +void +dl_error_set_remote_code(int remote_code) +{ + dl_error_detail.remote_code = remote_code; +} + +void +dl_error_set_stack(const char *stack) +{ + dl_error_copy_field(dl_error_detail.stack, + sizeof(dl_error_detail.stack), stack); +} + +const DlErrorDetail * +dl_error_get(void) +{ + return &dl_error_detail; +} + +const char * +dl_err_message(DlErrCode code) +{ + switch (code) + { + case DL_OK: + return "success"; + case DL_ERR_NOT_SUPPORTED: + return "operation not supported"; + case DL_ERR_INVALID_OPTION: + return "invalid option"; + case DL_ERR_NOT_FOUND: + return "not found"; + case DL_ERR_ALREADY_EXISTS: + return "already exists"; + case DL_ERR_IO: + return "I/O error"; + case DL_ERR_INTERNAL: + return "internal error"; + } + + return "unknown error"; +} + +void +dl_error_report(int elevel, DlErrCode code, const char *prefix) +{ + const DlErrorDetail *detail = dl_error_get(); + StringInfoData detail_buf; + bool has_detail; + + /* + * Detail recorded against a different code belongs to some other failure -- + * an implementation that reported this one without recording anything, for + * instance. Reporting it here would attribute the wrong cause. + */ + has_detail = (detail->code == code && + (detail->message[0] != '\0' || + detail->type[0] != '\0' || + detail->remote_code != 0)); + + if (!has_detail) + { + ereport(elevel, + (errcode(dl_error_sqlstate(code)), + errmsg("iceberg: %s failed: %s", prefix, + dl_err_message(code)))); + return; + } + + initStringInfo(&detail_buf); + + if (detail->operation[0] != '\0') + appendStringInfo(&detail_buf, "%s: ", detail->operation); + + if (detail->message[0] != '\0') + appendStringInfoString(&detail_buf, detail->message); + else + appendStringInfoString(&detail_buf, dl_err_message(code)); + + if (detail->type[0] != '\0') + appendStringInfo(&detail_buf, " (%s", detail->type); + if (detail->remote_code != 0) + appendStringInfo(&detail_buf, "%s%d", + detail->type[0] != '\0' ? ", code " : " (code ", + detail->remote_code); + if (detail->type[0] != '\0' || detail->remote_code != 0) + appendStringInfoChar(&detail_buf, ')'); + + /* + * A stack describes the implementation, not the statement, so it is offered + * only to a session that asked to see log-level detail. + */ + if (detail->stack[0] != '\0' && client_min_messages <= LOG) + appendStringInfo(&detail_buf, "\nStack:\n%s", detail->stack); + + ereport(elevel, + (errcode(dl_error_sqlstate(code)), + errmsg("iceberg: %s failed", prefix), + errdetail("%s", detail_buf.data))); + + pfree(detail_buf.data); +} diff --git a/contrib/datalake_fdw/src/common/dl_err.h b/contrib/datalake_fdw/src/common/dl_err.h new file mode 100644 index 00000000000..67949644c95 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_err.h @@ -0,0 +1,120 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_err.h + * Error codes shared by the layers below the access method, and the + * channel that carries what the code alone cannot say. + * + * A code says which kind of failure occurred. It cannot say which table the + * remote catalog rejected, what the storage service answered, or where a remote + * implementation threw -- and those are the only things that make such a failure + * diagnosable. Every layer below the access method therefore reports a code and + * additionally records the detail here; the entry points that face PostgreSQL + * turn both into one ereport. + * + * The detail is recorded into fixed-size storage on purpose. Recording happens + * on paths that must not allocate and must not raise -- a cleanup callback + * crossing back from C++ is one of them -- so a setter that could palloc, and + * therefore could fail, is not usable there. Long values are truncated, which + * is the right trade against losing the report entirely. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_err.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_ERR_H +#define DL_ERR_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum DlErrCode { + DL_OK = 0, + DL_ERR_NOT_SUPPORTED, + DL_ERR_INVALID_OPTION, + DL_ERR_NOT_FOUND, + DL_ERR_ALREADY_EXISTS, + DL_ERR_IO, + DL_ERR_INTERNAL, +} DlErrCode; + +#define DL_ERR_FIELD_LEN 128 +#define DL_ERR_MSG_LEN 1024 +#define DL_ERR_STACK_LEN 4096 + +/* + * What an implementation below the access method has to say about its last + * failure. The field set follows what a remote metadata engine reports, so + * that connecting one is a matter of filling this in rather than changing it. + */ +typedef struct DlErrorDetail +{ + DlErrCode code; + int remote_code; /* implementation's own numeric code, 0 + * when it has none */ + char operation[DL_ERR_FIELD_LEN]; /* which call failed */ + char type[DL_ERR_FIELD_LEN]; /* implementation's error class */ + char message[DL_ERR_MSG_LEN]; + char stack[DL_ERR_STACK_LEN]; +} DlErrorDetail; + +/* + * Discard any recorded detail. Called by the dispatch wrappers before entering + * an implementation, so that a report can never describe an earlier failure. + */ +extern void dl_error_reset(void); + +/* + * Record the detail for a failure that is being reported as `code`. Allocates + * nothing and raises nothing, so it is callable from a cleanup path. NULL is + * accepted for any string and leaves that field empty. + */ +extern void dl_error_set(DlErrCode code, const char *operation, + const char *type, const char *message); + +/* Record the implementation's own numeric code, when it reports one. */ +extern void dl_error_set_remote_code(int remote_code); + +/* Record a stack from the failing implementation. */ +extern void dl_error_set_stack(const char *stack); + +/* The recorded detail, never NULL; code is DL_OK when nothing was recorded. */ +extern const DlErrorDetail *dl_error_get(void); + +/* Short, code-only wording, for when there is nothing recorded to add. */ +extern const char *dl_err_message(DlErrCode code); + +/* + * Report a failure to PostgreSQL: `prefix` names what was being attempted, the + * recorded message becomes the detail, and a recorded stack is included only + * when the session asked for log-level detail -- a stack is for whoever is + * debugging the implementation, not for whoever ran the statement. + * + * Detail recorded against a different code is ignored rather than misattributed. + */ +extern void dl_error_report(int elevel, DlErrCode code, const char *prefix); + +#ifdef __cplusplus +} +#endif + +#endif /* DL_ERR_H */ diff --git a/contrib/datalake_fdw/src/common/dl_kv.h b/contrib/datalake_fdw/src/common/dl_kv.h new file mode 100644 index 00000000000..f5db0db31d8 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_kv.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_kv.h + * A configuration pair as options and mappings deliver it. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_kv.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_KV_H +#define DL_KV_H + +/* + * A configuration pair as it arrives from a foreign server, a user mapping or + * a table option. It lives in common/ because both the storage layer and the + * metadata layer consume such pairs, and neither should have to include the + * other's headers to name the type. + */ +typedef struct DlKeyValue +{ + char *key; + char *value; +} DlKeyValue; + +#endif /* DL_KV_H */ diff --git a/contrib/datalake_fdw/src/common/dl_option_util.c b/contrib/datalake_fdw/src/common/dl_option_util.c new file mode 100644 index 00000000000..c2cfbdaef54 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_option_util.c @@ -0,0 +1,65 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_option_util.c + * Option policy shared by the catalog and volume option validators. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_option_util.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/dl_option_util.h" + +bool +dl_is_credential_option(const char *name) +{ + static const char *const credential_options[] = { + DL_OPTION_KEY_USERNAME, + DL_OPTION_KEY_KRB_CLIENT_KEYTAB, + DL_OPTION_KEY_CLIENT_ID, + DL_OPTION_KEY_CLIENT_SECRET, + DL_OPTION_KEY_ACCESS_KEY_ID, + DL_OPTION_KEY_SECRET_ACCESS_KEY, + DL_OPTION_KEY_SESSION_TOKEN, + + /* + * Not options this module accepts anywhere, but names users reach for + * out of habit. Listing them turns "unrecognized option" into the hint + * that says where credentials actually go. + */ + "user", + "password", + "token", + "access_key", + "secret_key" + }; + int i; + + for (i = 0; i < lengthof(credential_options); i++) + { + if (strcmp(name, credential_options[i]) == 0) + return true; + } + + return false; +} diff --git a/contrib/datalake_fdw/src/common/dl_option_util.h b/contrib/datalake_fdw/src/common/dl_option_util.h new file mode 100644 index 00000000000..2ac33fadf77 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_option_util.h @@ -0,0 +1,63 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_option_util.h + * Option policy shared by the catalog and volume option validators. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_option_util.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_OPTION_UTIL_H +#define DL_OPTION_UTIL_H + +#include "postgres.h" + +/* + * The option keys that identify or authenticate a principal. + * + * They are defined here, below the option modules that name them, because + * dl_is_credential_option() and those modules have to agree: a key the modules + * accept but this list does not know is a credential the server would store in + * the clear. One definition each is what makes disagreement impossible; the + * per-module macros below are spelled the way the reference implementation + * spells them and resolve to these. + */ +#define DL_OPTION_KEY_USERNAME "username" +#define DL_OPTION_KEY_KRB_CLIENT_KEYTAB "krb_client_keytab" +#define DL_OPTION_KEY_CLIENT_ID "client_id" +#define DL_OPTION_KEY_CLIENT_SECRET "client_secret" +#define DL_OPTION_KEY_ACCESS_KEY_ID "access_key_id" +#define DL_OPTION_KEY_SECRET_ACCESS_KEY "secret_access_key" +#define DL_OPTION_KEY_SESSION_TOKEN "session_token" + +/* + * True for an option that identifies or authenticates a principal. Such an + * option belongs to a user mapping, never to a server, so that one server can + * be shared by roles with different credentials and so that the value is not + * readable through pg_foreign_server by every role holding USAGE. + * + * Both option validators consult this, which is why it lives here rather than + * being spelled out twice. + */ +extern bool dl_is_credential_option(const char *name); + +#endif /* DL_OPTION_UTIL_H */ diff --git a/contrib/datalake_fdw/src/common/dl_pg_api.h b/contrib/datalake_fdw/src/common/dl_pg_api.h new file mode 100644 index 00000000000..6584db45fff --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_pg_api.h @@ -0,0 +1,45 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_pg_api.h + * The PostgreSQL headers, safe to include from C++. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_pg_api.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_PG_API_H +#define DL_PG_API_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "postgres.h" + +#include "access/xact.h" +#include "utils/elog.h" + +#ifdef __cplusplus +} +#endif + +#endif /* DL_PG_API_H */ diff --git a/contrib/datalake_fdw/src/common/dl_wrappers.h b/contrib/datalake_fdw/src/common/dl_wrappers.h new file mode 100644 index 00000000000..737d97fae7d --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_wrappers.h @@ -0,0 +1,193 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_wrappers.h + * The boundaries between C++ code and the PostgreSQL runtime. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_wrappers.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_WRAPPERS_H +#define DL_WRAPPERS_H + +#include + +#include "common/dl_err.h" +#include "common/dl_pg_api.h" + +/* + * Exception-boundary classes: + * + * 1. PG-called C entry (extern "C" handler/UDF/hook): + * DL_TRY { } DL_CATCH_END(); converts C++ exceptions to ereport(ERROR, + * ...) at the boundary; no C++ exception may escape into PG stack frames. + * 2. Status-returning DlErrCode ABI (vtable implementations): functions must + * be noexcept; DL_ABI_GUARD_BEGIN / DL_ABI_GUARD_END(errvar) converts every + * exception to DL_ERR_INTERNAL. + * 3. Void cleanup ABI (close, abort, iterator close): functions must be + * noexcept and idempotent, and must never ereport. The cleanup guard logs + * a best-effort WARNING only outside error cleanup and otherwise swallows. + * 4. C++ calling PG APIs: DL_WRAP_START; ... DL_WRAP_END; converts a PG longjmp + * to DlPgError so the C++ caller can handle it without crossing the ABI. + */ + +#ifdef __cplusplus + +#include + +class DlPgError : public std::exception { +public: + const char *what() const noexcept override + { + return "PostgreSQL error"; + } +}; + +/* Save and restore PG's longjmp targets around a C++ call site. */ +class DlPgExceptionStack { +public: + DlPgExceptionStack(void **exception_stack, void **error_context_stack) + : exception_stack_(exception_stack), + error_context_stack_(error_context_stack), + saved_exception_stack_(*exception_stack), + saved_error_context_stack_(*error_context_stack) + { + } + + ~DlPgExceptionStack() + { + *exception_stack_ = saved_exception_stack_; + *error_context_stack_ = saved_error_context_stack_; + } + + void SetLocalJmp(void *local_jump) + { + *exception_stack_ = local_jump; + } + +private: + void **exception_stack_; + void **error_context_stack_; + void *saved_exception_stack_; + void *saved_error_context_stack_; +}; + +static inline bool +dl_can_log_cleanup_warning(void) +{ + return !in_error_recursion_trouble() && !IsAbortInProgress() && + !IsAbortedTransactionBlockState(); +} + +/* + * Class 1: a C entry point called by PostgreSQL. + * + * ereport(ERROR) unwinds with longjmp(), and longjmp()ing out of a C++ catch + * handler leaves the in-flight exception alive, which is undefined behavior. + * So the handler only records what happened -- the message is copied into a + * local buffer because the exception object dies with the handler -- and the + * ereport() happens after the try/catch statement has been left, the same way + * PAX defers it to CBDB_END_TRY(). + */ +#define DL_ERROR_MSG_MAX 512 + +#define DL_TRY \ + do { \ + bool dl_pending_error_ = false; \ + char dl_error_msg_[DL_ERROR_MSG_MAX]; \ +\ + dl_error_msg_[0] = '\0'; \ + try + +#define DL_CATCH_END() \ + catch (const std::exception &e) \ + { \ + dl_pending_error_ = true; \ + strlcpy(dl_error_msg_, e.what(), sizeof(dl_error_msg_)); \ + } \ + catch (...) \ + { \ + dl_pending_error_ = true; \ + strlcpy(dl_error_msg_, "unknown C++ exception", \ + sizeof(dl_error_msg_)); \ + } \ + if (dl_pending_error_) \ + ereport(ERROR, \ + (errcode(ERRCODE_INTERNAL_ERROR), \ + errmsg("datalake_fdw: %s", dl_error_msg_))); \ + } while (0) + +#define DL_ABI_GUARD_BEGIN \ + try \ + { + +#define DL_ABI_GUARD_END(errvar) \ + } \ + catch (...) \ + { \ + (errvar) = DL_ERR_INTERNAL; \ + } + +#define DL_CLEANUP_GUARD_BEGIN \ + do { \ + bool dl_cleanup_failed_ = false; \ +\ + try \ + { + +/* + * Class 3 cleanup guards must never ereport(), so the report is a WARNING at + * most -- and it is emitted after the handler has been left, because even + * elog() can escalate into an ERROR while the error subsystem is in trouble. + */ +#define DL_CLEANUP_GUARD_END \ + } \ + catch (...) \ + { \ + dl_cleanup_failed_ = true; \ + } \ + if (dl_cleanup_failed_ && dl_can_log_cleanup_warning()) \ + elog(WARNING, "datalake_fdw: C++ exception during cleanup"); \ + } while (0) + +/* Modeled on the PAX CBDB_WRAP_START/END saved-exception-stack pattern. */ +#define DL_WRAP_START \ + sigjmp_buf dl_local_sigjmp_buf; \ + { \ + DlPgExceptionStack dl_exception_stack( \ + reinterpret_cast(&PG_exception_stack), \ + reinterpret_cast(&error_context_stack)); \ + if (sigsetjmp(dl_local_sigjmp_buf, 0) == 0) \ + { \ + dl_exception_stack.SetLocalJmp(&dl_local_sigjmp_buf) + +#define DL_WRAP_END \ + } \ + else \ + { \ + throw DlPgError(); \ + } \ + } + +#endif /* __cplusplus */ + +#endif /* DL_WRAPPERS_H */ diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp new file mode 100644 index 00000000000..2b576fe4344 --- /dev/null +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -0,0 +1,232 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * file_system_wrapper.cpp + * Storage facade dispatching to the registered backend. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/file_system_wrapper.cpp + * + *------------------------------------------------------------------------- + */ + +#include "common/dl_pg_api.h" + +#include "common/backend_registry.h" +#include "common/dl_wrappers.h" +#include "common/file_system_wrapper.h" + +/* + * Dispatch only: each call finds the backend registered for the location's + * scheme and hands the work over. Nothing here is reachable from SQL in this + * skeleton, so what the regression suite asserts is the behaviour of the + * layers above. + */ + +extern "C" DlErrCode +datalake_fs_open(const DatalakeLocation *location, + const DlKeyValue *credentials, int ncredentials, + DatalakeFileSystem *fs_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + const struct DatalakeStorageOps *ops; + + if (fs_out == NULL || location == NULL || location->scheme == NULL || + ncredentials < 0) + rc = DL_ERR_INVALID_OPTION; + else + { + *fs_out = NULL; + ops = datalake_lookup_storage_backend(location->scheme); + + if (ops == NULL) + rc = DL_ERR_NOT_SUPPORTED; + else + { + rc = ops->fs_open(location, credentials, ncredentials, fs_out); + + if (rc == DL_OK && *fs_out == NULL) + rc = DL_ERR_INTERNAL; + else if (rc == DL_OK) + (*fs_out)->ops = ops; + } + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" void +datalake_fs_close(DatalakeFileSystem *fs) +{ + DL_CLEANUP_GUARD_BEGIN + { + /* + * Clear the caller's handle before releasing it, so that a repeated + * close -- the normal shape of resource-owner cleanup after an error + * that already closed things -- finds nothing to do instead of + * reaching a backend that has freed itself. + */ + if (fs != NULL && *fs != NULL) + { + DatalakeFileSystem doomed = *fs; + + *fs = NULL; + doomed->ops->fs_close(doomed); + } + } + DL_CLEANUP_GUARD_END; +} + +extern "C" DlErrCode +datalake_fs_list(DatalakeFileSystem fs, const char *prefix, + char ***names_out, int *nnames_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (fs == NULL || prefix == NULL || names_out == NULL || + nnames_out == NULL) + rc = DL_ERR_INVALID_OPTION; + else + { + *names_out = NULL; + *nnames_out = 0; + rc = fs->ops->fs_list(fs, prefix, names_out, nnames_out); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_open(DatalakeFileSystem fs, const char *path, + DatalakeFileMode mode, DatalakeFile *file_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file_out == NULL || fs == NULL || path == NULL) + rc = DL_ERR_INVALID_OPTION; + else if (mode != DATALAKE_FILE_READ && mode != DATALAKE_FILE_WRITE) + rc = DL_ERR_INVALID_OPTION; + else + { + *file_out = NULL; + rc = fs->ops->file_open(fs, path, mode, file_out); + + if (rc == DL_OK && *file_out == NULL) + rc = DL_ERR_INTERNAL; + else if (rc == DL_OK) + (*file_out)->ops = fs->ops; + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_read(DatalakeFile file, void *buffer, int64_t length, + int64_t *nread) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file == NULL || nread == NULL || length < 0) + rc = DL_ERR_INVALID_OPTION; + else + { + *nread = 0; + rc = file->ops->file_read(file, buffer, length, nread); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_write(DatalakeFile file, const void *buffer, int64_t length) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file == NULL || length < 0) + rc = DL_ERR_INVALID_OPTION; + else + rc = file->ops->file_write(file, buffer, length); + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_close(DatalakeFile *file) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file == NULL || *file == NULL) + rc = DL_ERR_INVALID_OPTION; + else + { + DatalakeFile doomed = *file; + + /* + * The handle is consumed even when the close reports an error: + * the backend has released it either way, and there is nothing + * left to retry the close against. + */ + *file = NULL; + rc = doomed->ops->file_close(doomed); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" void +datalake_file_abort(DatalakeFile *file) +{ + DL_CLEANUP_GUARD_BEGIN + { + /* Cleared first, so a repeated abort finds nothing to do. */ + if (file != NULL && *file != NULL) + { + DatalakeFile doomed = *file; + + *file = NULL; + doomed->ops->file_abort(doomed); + } + } + DL_CLEANUP_GUARD_END; +} diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.h b/contrib/datalake_fdw/src/common/file_system_wrapper.h new file mode 100644 index 00000000000..37b6c037d2c --- /dev/null +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.h @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * file_system_wrapper.h + * Storage facade over one protocol: open, read, write, list. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/file_system_wrapper.h + * + *------------------------------------------------------------------------- + */ + +#ifndef FILE_SYSTEM_WRAPPER_H +#define FILE_SYSTEM_WRAPPER_H + +#include + +#include "common/datalake_location.h" +#include "common/dl_err.h" +#include "common/dl_kv.h" + +/* + * A file system reached over one storage protocol, and an open file in it. + * Both are opaque: callers hold a handle and pass it back, exactly as they do + * for a File or a BufFile, so a backend can keep whatever state it needs + * without any of it becoming part of this interface. + * + * This is deliberately a facade over open/read/write/close/list and not a + * storage framework. Its only consumer is the format layer, and keeping the + * surface this narrow is what lets the implementation be replaced -- by an + * Arrow filesystem, say -- without the layers above noticing. + */ +typedef struct DatalakeFileSystemData *DatalakeFileSystem; +typedef struct DatalakeFileData *DatalakeFile; + +typedef enum DatalakeFileMode +{ + DATALAKE_FILE_READ, + DATALAKE_FILE_WRITE +} DatalakeFileMode; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* + * The location names the protocol and the bucket or namenode; credentials are + * resolved separately and may be empty, in which case the backend falls back + * to whatever ambient credentials it finds. + */ +extern DlErrCode datalake_fs_open(const DatalakeLocation *location, + const DlKeyValue *credentials, + int ncredentials, + DatalakeFileSystem *fs_out); + +/* + * Cleanup entry point: releases the file system and clears the caller's + * handle, so a repeated call has nothing left to act on. It never raises, + * because it runs on the resource-owner path during transaction abort. + * Passing a handle by value could not clear it, and the second call would + * then reach a backend that had already freed itself. + */ +extern void datalake_fs_close(DatalakeFileSystem *fs); + +extern DlErrCode datalake_fs_list(DatalakeFileSystem fs, const char *prefix, + char ***names_out, int *nnames_out); + +extern DlErrCode datalake_file_open(DatalakeFileSystem fs, const char *path, + DatalakeFileMode mode, + DatalakeFile *file_out); + +extern DlErrCode datalake_file_read(DatalakeFile file, void *buffer, + int64_t length, int64_t *nread); + +extern DlErrCode datalake_file_write(DatalakeFile file, const void *buffer, + int64_t length); + +/* + * Finishes the file and clears the caller's handle. Errors worth reporting + * surface here, and the handle is consumed whether or not one does: there is + * nothing left to retry against. + */ +extern DlErrCode datalake_file_close(DatalakeFile *file); + +/* + * Cleanup entry point for the failure path: discards the file and clears the + * caller's handle. Never raises; anything worth reporting comes out of + * datalake_file_close() instead. + */ +extern void datalake_file_abort(DatalakeFile *file); + +#ifdef __cplusplus +} +#endif + +#endif /* FILE_SYSTEM_WRAPPER_H */ diff --git a/contrib/datalake_fdw/src/common/parser_option.c b/contrib/datalake_fdw/src/common/parser_option.c new file mode 100644 index 00000000000..9e4a4ab0469 --- /dev/null +++ b/contrib/datalake_fdw/src/common/parser_option.c @@ -0,0 +1,94 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parser_option.c + * Typed accessors over a DefElem option list. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/parser_option.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/defrem.h" +#include "common/parser_option.h" +#include "utils/builtins.h" + +/* + * Return the value of the named option, or NULL when it is absent. + * + * The comparison is case-sensitive, matching how the server stores and + * de-duplicates option names. Matching case-insensitively here would let + * "type" and a quoted "TYPE" both be stored -- the generic duplicate check + * would not see them as the same option -- and then silently return whichever + * came first, which for a credential is the wrong one to pick at random. + */ +char * +get_string_option(List *options, const char *option_name) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, option_name) == 0) + return defGetString(def); + } + + return NULL; +} + +/* + * Boolean accessor that also reports whether the option was written at all. + * + * Callers that forward options to the metadata engine need that distinction: + * an unset boolean may fall back to site configuration, while an explicit + * false has to override it. + * + * Unlike the reference implementation, an unparsable value raises an error + * instead of silently yielding the default -- a typo in a boolean server + * option should not read as "you asked for the default". Stored values are + * already validated by the option validators, so this only fires on input + * paths that have not been through them. + */ +bool +get_bool_option_ex(List *options, const char *option_name, + bool default_value, bool *isset) +{ + char *value = get_string_option(options, option_name); + bool parsed_value; + + Assert(isset != NULL); + *isset = false; + + if (value == NULL) + return default_value; + + if (!parse_bool(value, &parsed_value)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid boolean value \"%s\" for option \"%s\"", + value, option_name))); + + *isset = true; + return parsed_value; +} diff --git a/contrib/datalake_fdw/src/common/parser_option.h b/contrib/datalake_fdw/src/common/parser_option.h new file mode 100644 index 00000000000..10a8181e3e0 --- /dev/null +++ b/contrib/datalake_fdw/src/common/parser_option.h @@ -0,0 +1,54 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parser_option.h + * Typed accessors over a DefElem option list. + * + * Every layer that reads SERVER, USER MAPPING or table options goes through + * these accessors rather than walking the list itself, so that lookup and + * absent-versus-empty are decided in one place. + * + * The reference implementation -- the existing implementation of this feature + * that this work derives from and is meant to replace -- also carries integer + * and defaulting-boolean accessors (getIntOption, getBoolOption). They are + * omitted here rather than shipped unused; add them under those names, as thin + * wrappers over the accessors below, together with the first option that needs + * them. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/parser_option.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PARSER_OPTION_H +#define PARSER_OPTION_H + +#include "postgres.h" + +#include "nodes/pg_list.h" + +/* Reference implementation: getStringOption() */ +extern char *get_string_option(List *options, const char *option_name); + +/* Reference implementation: getBoolOptionEx() */ +extern bool get_bool_option_ex(List *options, const char *option_name, + bool default_value, bool *isset); + +#endif /* PARSER_OPTION_H */ diff --git a/contrib/datalake_fdw/src/common/s3_file_system.cpp b/contrib/datalake_fdw/src/common/s3_file_system.cpp new file mode 100644 index 00000000000..d255200528b --- /dev/null +++ b/contrib/datalake_fdw/src/common/s3_file_system.cpp @@ -0,0 +1,261 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * s3_file_system.cpp + * The S3 storage backend. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/s3_file_system.cpp + * + *------------------------------------------------------------------------- + */ + +#include "common/dl_pg_api.h" + +#include "common/backend_registry.h" +#include "common/dl_wrappers.h" + +#include + +/* + * The S3 backend, without an S3 client yet: the shape a backend takes is what + * this file establishes, so that the change adding a real client replaces + * method bodies rather than the structure around them. Every entry point + * reports that the operation is not supported. + */ +class S3FileSystem +{ +public: + DlErrCode + Initialize(const DatalakeLocation *location, const DlKeyValue *credentials, + int ncredentials) + { + (void) location; + (void) credentials; + (void) ncredentials; + + return DL_ERR_NOT_SUPPORTED; + } + + DlErrCode + OpenFile(const char *path, DatalakeFileMode mode, DatalakeFile *file_out) + { + (void) path; + (void) mode; + + if (file_out != NULL) + *file_out = NULL; + + return DL_ERR_NOT_SUPPORTED; + } + + DlErrCode + List(const char *prefix, char ***names_out, int *nnames_out) + { + (void) prefix; + + if (names_out != NULL) + *names_out = NULL; + if (nnames_out != NULL) + *nnames_out = 0; + + return DL_ERR_NOT_SUPPORTED; + } +}; + +/* + * A handle the facade can hold. + * + * Deriving from the C struct rather than embedding it as a first member is what + * makes recovering the handle defined behaviour: a derived-to-base pointer + * conversion and a static_cast back are guaranteed for any class, while the + * first-member trick is only guaranteed for standard-layout types -- which this + * is not, because of the unique_ptr. The C side still sees a plain + * DatalakeFileSystemData, since that is what the base subobject is. + */ +struct S3FileSystemHandle : public DatalakeFileSystemData +{ + std::unique_ptr impl; +}; + +static DlErrCode +s3_fs_open(const DatalakeLocation *location, const DlKeyValue *credentials, + int ncredentials, DatalakeFileSystem *fs_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (fs_out == NULL) + rc = DL_ERR_INVALID_OPTION; + else + { + /* + * Owned by unique_ptr until the handle is published, so that an + * exception from the second allocation or from Initialize() -- + * which the guard below turns into an error code -- cannot leave + * the first allocation behind. + */ + std::unique_ptr handle(new S3FileSystemHandle()); + + *fs_out = NULL; + handle->impl.reset(new S3FileSystem()); + rc = handle->impl->Initialize(location, credentials, ncredentials); + + if (rc == DL_OK) + *fs_out = handle.release(); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static void +s3_fs_close(DatalakeFileSystem fs) +{ + DL_CLEANUP_GUARD_BEGIN + { + /* The facade has already cleared its caller's handle. */ + delete static_cast(fs); + } + DL_CLEANUP_GUARD_END; +} + +static DlErrCode +s3_fs_list(DatalakeFileSystem fs, const char *prefix, char ***names_out, + int *nnames_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + S3FileSystemHandle *handle = static_cast(fs); + + if (handle == NULL) + rc = DL_ERR_INVALID_OPTION; + else + rc = handle->impl->List(prefix, names_out, nnames_out); + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_open(DatalakeFileSystem fs, const char *path, DatalakeFileMode mode, + DatalakeFile *file_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + S3FileSystemHandle *handle = static_cast(fs); + + if (handle == NULL) + rc = DL_ERR_INVALID_OPTION; + else + rc = handle->impl->OpenFile(path, mode, file_out); + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_read(DatalakeFile file, void *buffer, int64_t length, int64_t *nread) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + (void) file; + (void) buffer; + (void) length; + + if (nread != NULL) + *nread = 0; + + rc = DL_ERR_NOT_SUPPORTED; + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_write(DatalakeFile file, const void *buffer, int64_t length) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + (void) file; + (void) buffer; + (void) length; + + rc = DL_ERR_NOT_SUPPORTED; + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_close(DatalakeFile file) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + (void) file; + + rc = DL_ERR_NOT_SUPPORTED; + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static void +s3_file_abort(DatalakeFile file) +{ + DL_CLEANUP_GUARD_BEGIN + { + (void) file; + } + DL_CLEANUP_GUARD_END; +} + +static const struct DatalakeStorageOps s3_storage_ops = { + s3_fs_open, + s3_fs_close, + s3_fs_list, + s3_file_open, + s3_file_read, + s3_file_write, + s3_file_close, + s3_file_abort +}; + +DlErrCode +datalake_register_s3_backend(void) +{ + return datalake_register_storage_backend("s3", &s3_storage_ops); +} diff --git a/contrib/datalake_fdw/src/format/format.h b/contrib/datalake_fdw/src/format/format.h new file mode 100644 index 00000000000..003101ae550 --- /dev/null +++ b/contrib/datalake_fdw/src/format/format.h @@ -0,0 +1,117 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * format.h + * Reader and writer interfaces for lake table data files. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/format.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_FORMAT_H +#define DL_FORMAT_H + +#include +#include + +#include "common/dl_err.h" + +/* Arrow C data interface: stable public ABI. */ +#ifndef ARROW_C_DATA_INTERFACE +#define ARROW_C_DATA_INTERFACE + +struct ArrowSchema { + const char *format; + const char *name; + const char *metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema **children; + struct ArrowSchema *dictionary; + void (*release)(struct ArrowSchema *); + void *private_data; +}; + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void **buffers; + struct ArrowArray **children; + struct ArrowArray *dictionary; + void (*release)(struct ArrowArray *); + void *private_data; +}; +#endif /* ARROW_C_DATA_INTERFACE */ + +typedef struct Fragment Fragment; /* opaque in skeleton */ +typedef struct ProjectionSet ProjectionSet; +typedef struct RowGroupFilterSet RowGroupFilterSet; +typedef struct WriterOptions WriterOptions; +typedef struct FileMeta FileMeta; +typedef struct DeleteFileSet DeleteFileSet; + +/* Readers/writers are INSTANCES (ops + impl); configuration travels with the instance. + * No global slots or trampolines, ever. */ +typedef struct FormatReader FormatReader; +typedef struct FormatReaderOps { + /* Each batch yields ArrowArray+ArrowSchema; last column is a hidden int64 file-row + * ordinal (for MoR positional deletes). */ + DlErrCode (*next_batch)(FormatReader *, struct ArrowArray *out, + struct ArrowSchema *schema, bool *eof); + void (*close)(FormatReader *); /* void cleanup ABI: noexcept, idempotent, never ereport */ +} FormatReaderOps; +struct FormatReader { const FormatReaderOps *ops; void *impl; }; + +typedef struct FormatWriter FormatWriter; +typedef struct FormatWriterOps { + DlErrCode (*write_batch)(FormatWriter *, struct ArrowArray *batch); /* success == consumed */ + /* Rolling support: actual bytes encoded into the sink so far. Valid to query after a + * successful write_batch; on failure returns an error code and *out is invalid. + * The write.c orchestration layer rolls files (finish -> new open_writer) when this + * reaches the soft target; overshoot of at most one batch is allowed. */ + DlErrCode (*bytes_written)(FormatWriter *, int64_t *out); + DlErrCode (*finish)(FormatWriter *, FileMeta **meta); /* reportable close-time errors + * surface ONLY here */ + void (*abort)(FormatWriter *); /* void cleanup ABI: noexcept, idempotent, never ereport */ +} FormatWriterOps; +struct FormatWriter { const FormatWriterOps *ops; void *impl; }; + +typedef struct FormatRoutine { + uint32_t abi_version, struct_size; /* same prefix-compat semantics as meta engine */ + const char *name; /* "parquet" */ + DlErrCode (*open_reader)(const Fragment *, const ProjectionSet *, + const RowGroupFilterSet *, FormatReader **out); + DlErrCode (*open_writer)(const char *path, /* TupleDesc */ void *tupdesc, + const WriterOptions *, FormatWriter **out); +} FormatRoutine; + +extern const FormatRoutine *GetFormatRoutine(const char *format); + +/* MoR positional-delete decorator: consumes the inner instance, returns a new instance. + * close(outer) exactly-once: releases itself then close(inner); idempotent; on open + * failure the wrapper owns releasing inner. */ +extern DlErrCode WrapPositionDeleteFilter(FormatReader *inner, const DeleteFileSet *, + FormatReader **out); + +#endif /* DL_FORMAT_H */ diff --git a/contrib/datalake_fdw/src/format/format_registry.c b/contrib/datalake_fdw/src/format/format_registry.c new file mode 100644 index 00000000000..83370a51f64 --- /dev/null +++ b/contrib/datalake_fdw/src/format/format_registry.c @@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * format_registry.c + * Lookup of the reader and writer for a data file format. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/format_registry.c + * + *------------------------------------------------------------------------- + */ + +#include + +#include "format/format.h" + +/* No formats in the skeleton; parquet lands in PR-3/4. Callers must treat + * NULL as not-supported. */ +const FormatRoutine * +GetFormatRoutine(const char *format) +{ + return NULL; +} + +DlErrCode +WrapPositionDeleteFilter(FormatReader *inner, const DeleteFileSet *delete_files, + FormatReader **out) +{ + if (out != NULL) + *out = NULL; + return DL_ERR_NOT_SUPPORTED; +} diff --git a/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c new file mode 100644 index 00000000000..ba408c6ab0c --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c @@ -0,0 +1,226 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_catalog_fdw.c + * Option validator for Iceberg catalog foreign servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/reloptions.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_user_mapping.h" +#include "commands/defrem.h" +#include "common/dl_option_util.h" +#include "common/parser_option.h" +#include "fmgr.h" +#include "iceberg_catalog_fdw/iceberg_catalog_option.h" + +PG_FUNCTION_INFO_V1(iceberg_catalog_fdw_validator); + +static bool is_catalog_server_option(const char *name); +static bool is_catalog_user_mapping_option(const char *name); +static void check_catalog_server_type(const char *server_type); + +/* + * The server options this wrapper accepts. A storage protocol is never among + * them: where the data files live is decided by the volume server. + */ +static bool +is_catalog_server_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_CATALOG_SERVER_TYPE) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_URL) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_NAME) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM) == 0; +} + +/* + * The user mapping options this wrapper accepts: the union over catalog types, + * because a mapping is validated without reference to the server it belongs to. + * parse_iceberg_catalog_user_mapping_options() is what narrows them by type. + */ +static bool +is_catalog_user_mapping_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_CATALOG_USERNAME) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_AUTH_METHOD) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_KRB_SERVICE_PRINCIPAL) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_PRINCIPAL) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_KEYTAB) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_CLIENT_ID) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_CLIENT_SECRET) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_SCOPE) == 0; +} + +static void +check_catalog_server_type(const char *server_type) +{ + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN) == 0) + return; + + /* + * Names the vocabulary defines but this module cannot serve yet. Refusing + * them is what keeps a server from being created against a catalog no + * statement could subsequently use. + */ + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HADOOP) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_S3) == 0) + pg_iceberg_not_supported(psprintf("catalog type \"%s\"", server_type)); + + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid iceberg catalog type \"%s\"", server_type), + errhint("Allowed types are \"%s\", \"%s\" and \"%s\"; \"%s\" is accepted as an alias of \"%s\".", + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST))); +} + +Datum +iceberg_catalog_fdw_validator(PG_FUNCTION_ARGS) +{ + List *options = untransformRelOptions(PG_GETARG_DATUM(0)); + Oid catalog = PG_GETARG_OID(1); + ListCell *lc; + const char *server_type; + const char *url; + const char *realm; + + /* + * CREATE FOREIGN DATA WRAPPER invokes its validator with an empty array. + * Permit that bootstrap call, but this FDW has no wrapper-level options. + */ + if (catalog == ForeignDataWrapperRelationId && options == NIL) + PG_RETURN_VOID(); + + if (catalog != ForeignServerRelationId && + catalog != UserMappingRelationId) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("iceberg_catalog_fdw has no options in this context"))); + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + const char *name = def->defname; + + if (catalog == ForeignServerRelationId) + { + if (dl_is_credential_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("credential option \"%s\" is not allowed on an iceberg catalog server", + name), + errhint("credentials belong in CREATE USER MAPPING ... OPTIONS (...)"))); + + if (!is_catalog_server_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg catalog server option \"%s\"", + name), + errhint("Allowed options are \"%s\", \"%s\", \"%s\", \"%s\" and \"%s\".", + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE, + DATALAKE_ICEBERG_CATALOG_URL, + DATALAKE_ICEBERG_CATALOG_NAME, + DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX, + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM))); + + /* Reject an empty value here rather than at first use. */ + if (defGetString(def)[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog server option \"%s\" cannot be empty", + name))); + } + else if (!is_catalog_user_mapping_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg catalog user mapping option \"%s\"", + name))); + } + + if (catalog == UserMappingRelationId) + PG_RETURN_VOID(); + + /* + * Cross-option rules run once the whole list has been seen, so that they do + * not depend on the order the options were written in. + */ + server_type = get_string_option(options, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE); + url = get_string_option(options, DATALAKE_ICEBERG_CATALOG_URL); + realm = get_string_option(options, + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM); + + if (server_type == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog server option \"%s\" is required", + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE))); + + check_catalog_server_type(server_type); + + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN) == 0) + { + if (url != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog type \"%s\" forbids server option \"%s\"", + server_type, DATALAKE_ICEBERG_CATALOG_URL))); + } + else if (url == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog type \"%s\" requires server option \"%s\"", + server_type, DATALAKE_ICEBERG_CATALOG_URL))); + + if (realm != NULL && + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) != 0 && + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) != 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog server option \"%s\" applies only to catalog type \"%s\"", + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST))); + + PG_RETURN_VOID(); +} diff --git a/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c new file mode 100644 index 00000000000..9587132ecba --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c @@ -0,0 +1,185 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_catalog_option.c + * Option vocabulary and parsed forms for Iceberg catalog servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/parser_option.h" +#include "iceberg_catalog_fdw/iceberg_catalog_option.h" +#include "utils/builtins.h" + +static void parse_hive_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options); +static void parse_rest_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options); +static void parse_hive_user_mapping_options(HiveUserMappingOptions *options, + List *user_options); +static void parse_polaris_user_mapping_options(PolarisUserMappingOptions *options, + List *user_options); + +/* + * Reference implementation: parseHiveCatalogServerOptions(). + * + * That version also accepts "hive_metastore_uri" as a second spelling of the + * same option, for servers created before the key was renamed. This extension + * has never been released, so there is nothing to be compatible with and only + * one spelling is accepted. + */ +static void +parse_hive_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options) +{ + options->hive_metastore_uri = + get_string_option(server_options, DATALAKE_ICEBERG_CATALOG_URL); +} + +/* Reference implementation: parsePolarisCatalogServerOptions() */ +static void +parse_rest_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options) +{ + options->polaris_server_url = + get_string_option(server_options, DATALAKE_ICEBERG_CATALOG_URL); + + /* + * Sent as the realm header on every request. Optional: the metadata engine + * applies its own default when unset. + */ + options->polaris_server_realm = + get_string_option(server_options, + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM); +} + +void +parse_iceberg_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options) +{ + options->server_type = + get_string_option(server_options, DATALAKE_ICEBERG_CATALOG_SERVER_TYPE); + + if (options->server_type == NULL) + return; + + if (pg_strcasecmp(options->server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE) == 0) + parse_hive_catalog_server_options(options, server_options); + else if (pg_strcasecmp(options->server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) == 0 || + pg_strcasecmp(options->server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) == 0) + parse_rest_catalog_server_options(options, server_options); +} + +/* Reference implementation: parseHiveUserMappingOptions() */ +static void +parse_hive_user_mapping_options(HiveUserMappingOptions *options, + List *user_options) +{ + options->username = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_USERNAME); + options->auth_method = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_AUTH_METHOD); + options->krb_service_principal = + get_string_option(user_options, + DATALAKE_ICEBERG_CATALOG_KRB_SERVICE_PRINCIPAL); + options->krb_client_principal = + get_string_option(user_options, + DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_PRINCIPAL); + options->krb_client_keytab = + get_string_option(user_options, + DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_KEYTAB); +} + +/* Reference implementation: parsePolarisUserMappingOptions() */ +static void +parse_polaris_user_mapping_options(PolarisUserMappingOptions *options, + List *user_options) +{ + options->client_id = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_CLIENT_ID); + options->client_secret = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_CLIENT_SECRET); + options->scope = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_SCOPE); +} + +void +parse_iceberg_catalog_user_mapping_options(IcebergCatalogUserMappingOptions *options, + List *user_options, + const char *server_type) +{ + if (server_type == NULL) + return; + + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE) == 0) + parse_hive_user_mapping_options(&options->hive, user_options); + else if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) == 0) + parse_polaris_user_mapping_options(&options->polaris, user_options); +} + +/* + * Reference implementation: parseIcebergForeignCatalogOptions(). + * + * server_name supplies the default catalog name, because a catalog server here + * names exactly one Iceberg catalog. + */ +void +parse_iceberg_foreign_catalog_options(IcebergForeignCatalogOptions *options, + List *catalog_options, + const char *server_name) +{ + options->catalog_name = + get_string_option(catalog_options, DATALAKE_ICEBERG_CATALOG_NAME); + if (options->catalog_name == NULL) + options->catalog_name = pstrdup(server_name); + + options->warehouse_location_prefix = + get_string_option(catalog_options, + DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX); +} + +IcebergCatalogOptions * +get_iceberg_catalog_options(ForeignServer *server) +{ + IcebergCatalogOptions *options; + + Assert(server != NULL); + + options = (IcebergCatalogOptions *) palloc0(sizeof(IcebergCatalogOptions)); + + parse_iceberg_catalog_server_options(&options->catalog_server, + server->options); + parse_iceberg_foreign_catalog_options(&options->foreign_catalog, + server->options, + server->servername); + + return options; +} diff --git a/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h new file mode 100644 index 00000000000..df6e3b6b9a1 --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h @@ -0,0 +1,171 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_catalog_option.h + * Option vocabulary and parsed forms for Iceberg catalog servers. + * + * "The reference implementation", here and in the other option modules, means + * the existing implementation of this feature that this work derives from and + * is meant to replace. Its option key macros, struct names and field names are + * reproduced exactly, so that a parser for a further catalog type can move + * between the two as an addition rather than a rewrite. Where the two models + * genuinely differ, the difference is called out on the field. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h + * + *------------------------------------------------------------------------- + */ + +#ifndef ICEBERG_CATALOG_OPTION_H +#define ICEBERG_CATALOG_OPTION_H + +#include "postgres.h" + +#include "common/dl_option_util.h" +#include "foreign/foreign.h" +#include "nodes/pg_list.h" + +/* Catalog server options */ +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE "type" +#define DATALAKE_ICEBERG_CATALOG_URL "uri" +#define DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM "polaris_server_realm" + +/* + * Recognized catalog server types. + * + * The names are Apache Iceberg's, not this module's: a catalog reached over the + * REST protocol is "rest", because the specification defines one protocol that + * several implementations answer. "polaris" is accepted as an alias for it -- + * Polaris is one such implementation, and it is the spelling the reference + * implementation uses, so servers written for that one keep working. + * + * A storage protocol is never a catalog type -- where the data files live is + * volume business -- but the reference implementation defines these two names, + * so they are kept here to stay one vocabulary; the validator refuses them + * until an implementation exists. + */ +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE "hive" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST "rest" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS "polaris" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN "builtin" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HADOOP "hadoop" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_S3 "s3" + +/* Catalog user mapping options */ +#define DATALAKE_ICEBERG_CATALOG_USERNAME DL_OPTION_KEY_USERNAME +#define DATALAKE_ICEBERG_CATALOG_AUTH_METHOD "auth_method" +#define DATALAKE_ICEBERG_CATALOG_KRB_SERVICE_PRINCIPAL "krb_service_principal" +#define DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_PRINCIPAL "krb_client_principal" +#define DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_KEYTAB DL_OPTION_KEY_KRB_CLIENT_KEYTAB +#define DATALAKE_ICEBERG_CATALOG_CLIENT_ID DL_OPTION_KEY_CLIENT_ID +#define DATALAKE_ICEBERG_CATALOG_CLIENT_SECRET DL_OPTION_KEY_CLIENT_SECRET +#define DATALAKE_ICEBERG_CATALOG_SCOPE "scope" + +/* Catalog identity options */ +#define DATALAKE_ICEBERG_CATALOG_NAME "catalog_name" +#define DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX "warehouse" + +typedef struct IcebergCatalogServerOptions +{ + char *server_type; /* DATALAKE_ICEBERG_CATALOG_SERVER_TYPE */ + char *hive_metastore_uri; /* DATALAKE_ICEBERG_CATALOG_URL, hive */ + char *polaris_server_url; /* DATALAKE_ICEBERG_CATALOG_URL, rest */ + char *polaris_server_realm; /* DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM */ + + /* + * The reference implementation also carries server_name, naming a section + * of a site configuration file. There is no such file here: an option that + * would be accepted and then ignored is worse than one that is refused, so + * it is left out until site configuration exists. + */ +} IcebergCatalogServerOptions; + +typedef struct HiveUserMappingOptions +{ + char *username; + char *auth_method; + char *krb_service_principal; + char *krb_client_principal; + char *krb_client_keytab; +} HiveUserMappingOptions; + +typedef struct PolarisUserMappingOptions +{ + char *client_id; + char *client_secret; + char *scope; +} PolarisUserMappingOptions; + +typedef struct IcebergCatalogUserMappingOptions +{ + HiveUserMappingOptions hive; + PolarisUserMappingOptions polaris; +} IcebergCatalogUserMappingOptions; + +typedef struct IcebergForeignCatalogOptions +{ + /* + * The reference implementation reads these from a foreign catalog object + * that a server can hold several of. This extension cannot add a catalog + * of its own to the system catalogs, so a catalog server names exactly one + * Iceberg catalog and both values come from that server's options; + * catalog_name defaults to the server name when unset. + */ + char *catalog_name; /* DATALAKE_ICEBERG_CATALOG_NAME */ + char *warehouse_location_prefix; /* DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX */ + + /* + * Deferred, with the reference implementation's names kept for the port: + * enable_metadata_cache / metadata_cache_ttl / auto_refresh_metadata / + * total_segment / split_size / filter_string. + */ +} IcebergForeignCatalogOptions; + +typedef struct IcebergCatalogOptions +{ + IcebergCatalogServerOptions catalog_server; + IcebergCatalogUserMappingOptions catalog_user; + IcebergForeignCatalogOptions foreign_catalog; +} IcebergCatalogOptions; + +/* Reference implementation: parseIcebergCatalogServerOptions() */ +extern void parse_iceberg_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options); + +/* Reference implementation: parseIcebergCatalogUserMappingOptions() */ +extern void parse_iceberg_catalog_user_mapping_options(IcebergCatalogUserMappingOptions *options, + List *user_options, + const char *server_type); + +/* Reference implementation: parseIcebergForeignCatalogOptions() */ +extern void parse_iceberg_foreign_catalog_options(IcebergForeignCatalogOptions *options, + List *catalog_options, + const char *server_name); + +/* + * Reference implementation: getIcebergCatalogOptions(), which additionally + * fills catalog_user from the invoking role's user mapping. Credentials are + * resolved separately and lazily here, because a DDL path must be able to + * describe a table without reading anyone's secrets; catalog_user is left + * zeroed by this function. + */ +extern IcebergCatalogOptions *get_iceberg_catalog_options(ForeignServer *server); + +#endif /* ICEBERG_CATALOG_OPTION_H */ diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c new file mode 100644 index 00000000000..cff9f80f333 --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c @@ -0,0 +1,157 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_volume_fdw.c + * Option validator for Iceberg volume foreign servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/reloptions.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_user_mapping.h" +#include "commands/defrem.h" +#include "common/dl_option_util.h" +#include "fmgr.h" +#include "iceberg_volume_fdw/iceberg_volume_option.h" + +PG_FUNCTION_INFO_V1(iceberg_volume_fdw_validator); + +static bool is_volume_server_option(const char *name); +static bool is_volume_user_mapping_option(const char *name); + +static bool +is_volume_server_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_VOLUME_BASE_PATH) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_ENDPOINT) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_REGION) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS) == 0; +} + +/* + * Every credential here is optional, so that ambient storage credentials -- an + * instance profile, a ticket cache -- remain a valid deployment choice. + */ +static bool +is_volume_user_mapping_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_VOLUME_USERNAME) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN) == 0; +} + +Datum +iceberg_volume_fdw_validator(PG_FUNCTION_ARGS) +{ + List *options = untransformRelOptions(PG_GETARG_DATUM(0)); + Oid catalog = PG_GETARG_OID(1); + ListCell *lc; + IcebergVolumeServerOptions server_options; + IcebergForeignVolumeOptions volume_options; + DatalakeLocation location; + char *parse_detail = NULL; + DlErrCode parse_result; + + /* + * CREATE FOREIGN DATA WRAPPER invokes its validator with an empty array. + * Permit that bootstrap call, but this FDW has no wrapper-level options. + */ + if (catalog == ForeignDataWrapperRelationId && options == NIL) + PG_RETURN_VOID(); + + if (catalog != ForeignServerRelationId && + catalog != UserMappingRelationId) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("iceberg_volume_fdw has no options in this context"))); + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + const char *name = def->defname; + + if (catalog == ForeignServerRelationId) + { + if (dl_is_credential_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("credential option \"%s\" is not allowed on an iceberg volume server", + name), + errhint("credentials belong in CREATE USER MAPPING ... OPTIONS (...)"))); + + if (!is_volume_server_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg volume server option \"%s\"", + name), + errhint("Allowed options are \"%s\", \"%s\", \"%s\" and \"%s\".", + DATALAKE_ICEBERG_VOLUME_BASE_PATH, + DATALAKE_ICEBERG_VOLUME_ENDPOINT, + DATALAKE_ICEBERG_VOLUME_REGION, + DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS))); + } + else if (!is_volume_user_mapping_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg volume user mapping option \"%s\"", + name))); + } + + if (catalog == UserMappingRelationId) + PG_RETURN_VOID(); + + /* + * Parse with the same functions the use points parse with, so a server this + * validator accepted cannot fail to parse afterwards. path_style_access is + * checked as a side effect: the accessor refuses a non-boolean value. + */ + memset(&server_options, 0, sizeof(server_options)); + memset(&volume_options, 0, sizeof(volume_options)); + parse_iceberg_volume_server_options(&server_options, options); + parse_iceberg_foreign_volume_options(&volume_options, options); + + if (volume_options.base_path == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg volume server option \"%s\" is required", + DATALAKE_ICEBERG_VOLUME_BASE_PATH))); + + parse_result = pg_iceberg_parse_location(volume_options.base_path, + server_options.endpoint, + server_options.region, + &location, &parse_detail); + if (parse_result != DL_OK) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid iceberg volume %s \"%s\"", + DATALAKE_ICEBERG_VOLUME_BASE_PATH, + volume_options.base_path), + errdetail("%s", parse_detail))); + + PG_RETURN_VOID(); +} diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c new file mode 100644 index 00000000000..b304f27620e --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c @@ -0,0 +1,94 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_volume_option.c + * Option vocabulary and parsed forms for Iceberg volume servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/parser_option.h" +#include "iceberg_volume_fdw/iceberg_volume_option.h" + +void +parse_iceberg_volume_server_options(IcebergVolumeServerOptions *options, + List *server_options) +{ + options->endpoint = + get_string_option(server_options, DATALAKE_ICEBERG_VOLUME_ENDPOINT); + options->region = + get_string_option(server_options, DATALAKE_ICEBERG_VOLUME_REGION); + + /* + * Absence has to stay distinguishable from an explicit false: the metadata + * engine merges what a server states over its own defaults per key, so an + * unset boolean must not arrive as one the user chose. + */ + options->path_style_access = + get_bool_option_ex(server_options, + DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS, + false, &options->path_style_access_set); +} + +void +parse_iceberg_volume_user_mapping_options(IcebergVolumeUserMappingOptions *options, + List *user_options) +{ + options->username = + get_string_option(user_options, DATALAKE_ICEBERG_VOLUME_USERNAME); + options->aws_access_key_id = + get_string_option(user_options, + DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID); + options->aws_secret_access_key = + get_string_option(user_options, + DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY); + options->aws_session_token = + get_string_option(user_options, + DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN); +} + +void +parse_iceberg_foreign_volume_options(IcebergForeignVolumeOptions *options, + List *volume_options) +{ + options->base_path = + get_string_option(volume_options, DATALAKE_ICEBERG_VOLUME_BASE_PATH); +} + +IcebergVolumeOptions * +get_iceberg_volume_options(ForeignServer *server) +{ + IcebergVolumeOptions *options; + + Assert(server != NULL); + + options = (IcebergVolumeOptions *) palloc0(sizeof(IcebergVolumeOptions)); + + parse_iceberg_volume_server_options(&options->volume_server, + server->options); + parse_iceberg_foreign_volume_options(&options->foreign_volume, + server->options); + + return options; +} diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h new file mode 100644 index 00000000000..2b4ab16b46c --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h @@ -0,0 +1,146 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_volume_option.h + * Option vocabulary and parsed forms for Iceberg volume servers. + * + * As on the catalog side, the key macros, struct names and field names are + * reproduced from the reference implementation -- the existing implementation + * of this feature that this work derives from and is meant to replace -- so + * that support for a further storage protocol moves across as an addition. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h + * + *------------------------------------------------------------------------- + */ + +#ifndef ICEBERG_VOLUME_OPTION_H +#define ICEBERG_VOLUME_OPTION_H + +#include "postgres.h" + +#include "common/dl_option_util.h" +#include "foreign/foreign.h" +#include "nodes/pg_list.h" + +/* Volume server options */ +#define DATALAKE_ICEBERG_VOLUME_ENDPOINT "endpoint" +#define DATALAKE_ICEBERG_VOLUME_REGION "region" +#define DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS "path_style_access" + +/* Volume user mapping options */ +#define DATALAKE_ICEBERG_VOLUME_USERNAME DL_OPTION_KEY_USERNAME +#define DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID DL_OPTION_KEY_ACCESS_KEY_ID +#define DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY DL_OPTION_KEY_SECRET_ACCESS_KEY +#define DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN DL_OPTION_KEY_SESSION_TOKEN + +/* Volume location option */ +#define DATALAKE_ICEBERG_VOLUME_BASE_PATH "base_path" + +/* + * Storage protocols. Unlike the reference implementation, no server option + * names the protocol: base_path carries a URI, so its scheme already says which + * protocol this volume speaks, and a separate option could only disagree with + * it. The names are kept because the parsed location is compared against them. + */ +#define DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3 "s3" +#define DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS "hdfs" + +typedef struct IcebergVolumeServerOptions +{ + char *endpoint; /* DATALAKE_ICEBERG_VOLUME_ENDPOINT */ + char *region; /* DATALAKE_ICEBERG_VOLUME_REGION */ + bool path_style_access; /* DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS */ + bool path_style_access_set; /* user actually wrote path_style_access */ + + /* + * Deferred, with the reference implementation's names kept for the port. + * server_type and bucket_name are absent by design instead: both are + * derived from base_path, and DatalakeLocation is the parsed form. + * + * AWS: role_arn / external_id / user_arn / current_kms_key / + * allowed_kms_keys / sts_endpoint / sts_unavailable / endpoint_internal + * Azure: tenant_id / multi_tenant_app_name / consent_url / hierarchical + * HDFS: hdfs_namenodes / hdfs_port / hdfs_auth_method / krb_principal / + * krb_principal_keytab / krb_service_principal / + * hadoop_rpc_protection / data_transfer_protocol / is_ha_supported / + * dfs_nameservices / dfs_ha_namenodes / dfs_namenode_rpc_address / + * dfs_client_failover_proxy_provider / + * dfs_client_use_datanode_hostname + */ +} IcebergVolumeServerOptions; + +typedef struct IcebergVolumeUserMappingOptions +{ + char *username; /* DATALAKE_ICEBERG_VOLUME_USERNAME */ + char *aws_access_key_id; /* DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID */ + char *aws_secret_access_key; /* DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY */ + + /* + * Temporary credentials are three values, not two, and are what AWS + * recommends over long-lived keys; without this field the mapping could + * only express the long-lived form. + */ + char *aws_session_token; /* DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN */ +} IcebergVolumeUserMappingOptions; + +typedef struct IcebergForeignVolumeOptions +{ + /* + * The reference implementation reads these from a foreign volume object; as + * with the catalog side, a volume server here names exactly one volume and + * the value comes from that server's options. + * + * Deferred, names kept: enable_caching / allow_writes / fileIOConfig / + * table_identifier. + */ + char *base_path; /* DATALAKE_ICEBERG_VOLUME_BASE_PATH */ +} IcebergForeignVolumeOptions; + +typedef struct IcebergVolumeOptions +{ + IcebergVolumeServerOptions volume_server; + IcebergVolumeUserMappingOptions volume_user; + IcebergForeignVolumeOptions foreign_volume; +} IcebergVolumeOptions; + +/* Reference implementation: parseIcebergVolumeServerOptions() */ +extern void parse_iceberg_volume_server_options(IcebergVolumeServerOptions *options, + List *server_options); + +/* Reference implementation: parseIcebergVolumeUserMappingOptions() */ +extern void parse_iceberg_volume_user_mapping_options(IcebergVolumeUserMappingOptions *options, + List *user_options); + +/* Reference implementation: parseIcebergForeignVolumeOptions() */ +extern void parse_iceberg_foreign_volume_options(IcebergForeignVolumeOptions *options, + List *volume_options); + +/* + * Reference implementation: getIcebergVolumeOptions(). volume_user is left + * zeroed for the same reason as on the catalog side. + * + * The reference implementation's buildVolumeBasePath() has no counterpart: + * base_path is parsed once into a DatalakeLocation, and every layer below + * receives that instead of re-parsing a URI. + */ +extern IcebergVolumeOptions *get_iceberg_volume_options(ForeignServer *server); + +#endif /* ICEBERG_VOLUME_OPTION_H */ diff --git a/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c new file mode 100644 index 00000000000..dfd84c5e54e --- /dev/null +++ b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * stub_engine.c + * A metadata engine that reports what it would have done. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "meta/engine_stub/stub_engine.h" +#include "meta/iceberg_meta_engine.h" + +static DlErrCode +stub_create_table(const MetaCtx *ctx, const MetaTableDef *def, MetaTable **out) +{ + + ereport(NOTICE, + (errmsg("stub engine: created iceberg table \"%s.%s\" in catalog \"%s\"", + ctx->namespace_name, ctx->table_name, ctx->catalog_name))); + *out = NULL; + return DL_OK; +} + +static DlErrCode +stub_drop_table(const MetaCtx *ctx, bool purge) +{ + /* + * Report whether the data would have gone with it, so that a regression can + * see which of the two things a DROP asked for. + */ + ereport(NOTICE, + (errmsg("stub engine: dropped iceberg table \"%s.%s\" from catalog \"%s\"%s", + ctx->namespace_name, ctx->table_name, ctx->catalog_name, + purge ? ", purging data" : ", keeping data"))); + return DL_OK; +} + +static DlErrCode +stub_table_exists(const MetaCtx *ctx, bool *exists) +{ + + /* The stub has no remote catalog. */ + *exists = false; + return DL_OK; +} + +/* + * Nothing to load in the skeleton; the method remains a member of the lifecycle + * family, so it exists and refuses. + * + * It refuses the way a real engine has to: the code says what kind of failure it + * is, and everything specific to this failure -- which table, which operation, + * what the implementation calls it -- is recorded for the reporting layer. A + * remote engine records the message and stack it received here instead. + */ +static DlErrCode +stub_load_table(const MetaCtx *ctx, MetaTable **out) +{ + dl_error_set(DL_ERR_NOT_SUPPORTED, "load_table", "StubEngine", + psprintf("the stub engine holds no metadata for \"%s.%s\"", + ctx->namespace_name, ctx->table_name)); + return DL_ERR_NOT_SUPPORTED; +} + +static const IcebergMetaEngine stub_engine = +{ + .abi_version = DL_META_ENGINE_ABI_VERSION, + .struct_size = sizeof(IcebergMetaEngine), + .capabilities = DL_CAP_TABLE_LIFECYCLE, + .name = "stub", + .load_table = stub_load_table, + .create_table = stub_create_table, + .drop_table = stub_drop_table, + .table_exists = stub_table_exists +}; + +DlErrCode +RegisterStubMetaEngine(void) +{ + return RegisterMetaEngine(&stub_engine); +} diff --git a/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h new file mode 100644 index 00000000000..df87bb58d2e --- /dev/null +++ b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * stub_engine.h + * A metadata engine that reports what it would have done. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STUB_ENGINE_H +#define STUB_ENGINE_H + +#include "common/dl_err.h" + +extern DlErrCode RegisterStubMetaEngine(void); + +#endif /* STUB_ENGINE_H */ diff --git a/contrib/datalake_fdw/src/meta/iceberg_meta_engine.h b/contrib/datalake_fdw/src/meta/iceberg_meta_engine.h new file mode 100644 index 00000000000..ef80835eb27 --- /dev/null +++ b/contrib/datalake_fdw/src/meta/iceberg_meta_engine.h @@ -0,0 +1,149 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_meta_engine.h + * The metadata engine interface and its central dispatch. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/iceberg_meta_engine.h + * + *------------------------------------------------------------------------- + */ + +#ifndef ICEBERG_META_ENGINE_H +#define ICEBERG_META_ENGINE_H + +/* + * This is the header an engine implementation includes, and the next one is + * expected to be C++. Everything below therefore has to keep C linkage: a C++ + * translation unit that saw these as C++ declarations would emit mangled + * references and fail to link against the C registry. The server headers come + * in through dl_pg_api.h for the same reason. + */ +#include "common/dl_pg_api.h" + +#include "common/dl_err.h" +#include "common/dl_kv.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Server, mapping and table options all arrive as plain pairs. */ +typedef DlKeyValue MetaKv; + +typedef struct MetaCtx { /* identity + mapping, no credentials in skeleton */ + const char *catalog_name; /* catalog name within the external catalog */ + const char *namespace_name; /* PG schema name */ + const char *table_name; + const MetaKv *catalog_props; int n_catalog_props; + const MetaKv *credential_props; int n_credential_props; /* empty in stub paths */ +} MetaCtx; + +typedef struct MetaTableDef { const char *schema_json; } MetaTableDef; +typedef struct MetaTable { char *metadata_location; char *table_uuid; } MetaTable; +/* Opaque in skeleton: */ +typedef struct MetaStatistics MetaStatistics; +typedef struct MetaAppendRequest MetaAppendRequest; +typedef struct MetaCommitAppendRequest MetaCommitAppendRequest; +typedef struct MetaUpdateRequest MetaUpdateRequest; +typedef struct MetaCommitUpdateRequest MetaCommitUpdateRequest; +typedef struct MetaStageResult MetaStageResult; +typedef struct MetaCommitResult MetaCommitResult; +typedef struct MetaFilterExpr MetaFilterExpr; +typedef struct MetaFragmentIter MetaFragmentIter; +typedef struct MetaFragmentBatch MetaFragmentBatch; +typedef struct MetaFileGroupIter MetaFileGroupIter; +typedef struct MetaFileGroup MetaFileGroup; +typedef struct MetaFileGroupList MetaFileGroupList; +typedef struct MetaAlterTableRequest MetaAlterTableRequest; +typedef struct MetaTruncateRequest MetaTruncateRequest; + +#define DL_META_ENGINE_ABI_VERSION 1 + +typedef struct IcebergMetaEngine { + uint32_t abi_version; /* must equal DL_META_ENGINE_ABI_VERSION */ + uint32_t struct_size; /* PREFIX-compat: registry only touches fields covered by + * struct_size; validation bound is the minimal v1 prefix, + * NOT sizeof(current struct). Tail may only grow. */ + uint64_t capabilities; /* DL_CAP_* method-family bitmap, see below */ + const char *name; /* "agent" / "builtin" / "stub" */ + + DlErrCode (*load_table)(const MetaCtx *, MetaTable **); + DlErrCode (*create_table)(const MetaCtx *, const MetaTableDef *, MetaTable **); + DlErrCode (*drop_table)(const MetaCtx *, bool purge); + DlErrCode (*table_exists)(const MetaCtx *, bool *); + DlErrCode (*get_statistics)(const MetaCtx *, int64_t snapshot, MetaStatistics **); + + /* Iceberg single-table OCC: stage (append/update) then commit_*; NOT a cross-table + * distributed atomic commit. */ + DlErrCode (*append)(const MetaCtx *, const MetaAppendRequest *, MetaStageResult *); + DlErrCode (*commit_append)(const MetaCtx *, const MetaCommitAppendRequest *, MetaCommitResult *); + DlErrCode (*update)(const MetaCtx *, const MetaUpdateRequest *, MetaStageResult *); + DlErrCode (*commit_update)(const MetaCtx *, const MetaCommitUpdateRequest *, MetaCommitResult *); + + DlErrCode (*get_fragment)(const MetaCtx *, const char *metadata_location, + const MetaFilterExpr *, uint32_t batch_hint, MetaFragmentIter **); + DlErrCode (*plan_file_groups)(const MetaCtx *, const char *plan_options_json, MetaFileGroupIter **); + DlErrCode (*commit_file_groups)(const MetaCtx *, const MetaFileGroupList *, const char *, MetaCommitResult *); + DlErrCode (*alter_table)(const MetaCtx *, const MetaAlterTableRequest *, MetaCommitResult *); + DlErrCode (*truncate_table)(const MetaCtx *, const MetaTruncateRequest *, MetaCommitResult *); + + /* iterator close callbacks are "void cleanup ABI": noexcept, idempotent, never ereport */ + DlErrCode (*fragment_iter_next_batch)(MetaFragmentIter *, MetaFragmentBatch **); + void (*fragment_iter_close)(MetaFragmentIter *); + DlErrCode (*file_group_iter_next)(MetaFileGroupIter *, MetaFileGroup **); + void (*file_group_iter_close)(MetaFileGroupIter *); +} IcebergMetaEngine; + +#define DL_CAP_TABLE_LIFECYCLE (UINT64CONST(1) << 0) /* load_table, create_table, drop_table, table_exists */ +#define DL_CAP_STATISTICS (UINT64CONST(1) << 1) /* get_statistics */ +#define DL_CAP_APPEND (UINT64CONST(1) << 2) /* append, commit_append */ +#define DL_CAP_UPDATE (UINT64CONST(1) << 3) /* update, commit_update */ +#define DL_CAP_GET_FRAGMENT (UINT64CONST(1) << 4) /* get_fragment, fragment_iter_next_batch, fragment_iter_close */ +#define DL_CAP_REWRITE (UINT64CONST(1) << 5) /* plan_file_groups, commit_file_groups, file_group_iter_next, file_group_iter_close */ +#define DL_CAP_ALTER (UINT64CONST(1) << 6) /* alter_table */ +#define DL_CAP_TRUNCATE (UINT64CONST(1) << 7) /* truncate_table */ + +extern DlErrCode RegisterMetaEngine(const IcebergMetaEngine *engine); + +/* + * Returns the engine every lake table goes through. The engine is not + * selectable: nothing in a table's definition, and no configuration setting, + * picks between implementations, so a table can never be reinterpreted by a + * later change. The vtable indirection remains because the implementation + * behind it is expected to change -- the Java agent today, an in-process C++ + * one once it exists -- not because a deployment gets to choose. + */ +extern const IcebergMetaEngine *get_meta_engine(void); + +extern DlErrCode meta_engine_create_table(const IcebergMetaEngine *, const MetaCtx *, + const MetaTableDef *, MetaTable **); +extern DlErrCode meta_engine_drop_table(const IcebergMetaEngine *, const MetaCtx *, + bool purge); +extern DlErrCode meta_engine_table_exists(const IcebergMetaEngine *, const MetaCtx *, bool *); +extern DlErrCode meta_engine_load_table(const IcebergMetaEngine *, const MetaCtx *, MetaTable **); + +/* Remaining method families follow the same central-dispatch pattern in later PRs. */ + +#ifdef __cplusplus +} +#endif + +#endif /* ICEBERG_META_ENGINE_H */ diff --git a/contrib/datalake_fdw/src/meta/meta_engine_init.c b/contrib/datalake_fdw/src/meta/meta_engine_init.c new file mode 100644 index 00000000000..689b3d2c8d1 --- /dev/null +++ b/contrib/datalake_fdw/src/meta/meta_engine_init.c @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * meta_engine_init.c + * Registration of the metadata engine this build provides. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/meta_engine_init.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/dl_err.h" +#include "meta/engine_stub/stub_engine.h" +#include "meta/meta_engine_init.h" + +/* + * Called from _PG_init (extensible.c, later task); agent and builtin engines + * join in later PRs. SQL-visible NOTICE behavior is covered by skel-regress; + * this skeleton has no separate C test harness. + */ +void +DatalakeRegisterMetaEngines(void) +{ + DlErrCode rc; + + rc = RegisterStubMetaEngine(); + if (rc != DL_OK) + elog(ERROR, "datalake_fdw: failed to register stub meta engine: %s", + dl_err_message(rc)); +} diff --git a/contrib/datalake_fdw/src/meta/meta_engine_init.h b/contrib/datalake_fdw/src/meta/meta_engine_init.h new file mode 100644 index 00000000000..c20bc5fd6ba --- /dev/null +++ b/contrib/datalake_fdw/src/meta/meta_engine_init.h @@ -0,0 +1,34 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * meta_engine_init.h + * Registration of the metadata engine this build provides. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/meta_engine_init.h + * + *------------------------------------------------------------------------- + */ + +#ifndef META_ENGINE_INIT_H +#define META_ENGINE_INIT_H + +extern void DatalakeRegisterMetaEngines(void); + +#endif /* META_ENGINE_INIT_H */ diff --git a/contrib/datalake_fdw/src/meta/meta_engine_registry.c b/contrib/datalake_fdw/src/meta/meta_engine_registry.c new file mode 100644 index 00000000000..d4bb021c123 --- /dev/null +++ b/contrib/datalake_fdw/src/meta/meta_engine_registry.c @@ -0,0 +1,245 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * meta_engine_registry.c + * Metadata engine registry, capability checks and dispatch. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/meta_engine_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include +#include + +#include "meta/iceberg_meta_engine.h" + +/* Registered engine pointers are borrowed; each engine must have static lifetime. */ +static const IcebergMetaEngine *engines[8]; +static int n_engines; + +/* + * v1 minimal prefix = through file_group_iter_close; the registry must never + * touch fields beyond engine->struct_size. Do NOT compare against + * sizeof(IcebergMetaEngine) -- that breaks prefix compatibility once the struct + * grows. + */ +#define DL_META_ENGINE_V1_MIN_SIZE \ + (offsetof(IcebergMetaEngine, file_group_iter_close) + \ + sizeof(((IcebergMetaEngine *) 0)->file_group_iter_close)) + +typedef struct DlCapabilityFamily +{ + uint64 bit; + const char *family; + size_t offsets[5]; + int n; +} DlCapabilityFamily; + +static const DlCapabilityFamily capability_families[] = +{ + {DL_CAP_TABLE_LIFECYCLE, "table lifecycle", + {offsetof(IcebergMetaEngine, load_table), + offsetof(IcebergMetaEngine, create_table), + offsetof(IcebergMetaEngine, drop_table), + offsetof(IcebergMetaEngine, table_exists)}, 4}, + {DL_CAP_STATISTICS, "statistics", + {offsetof(IcebergMetaEngine, get_statistics)}, 1}, + {DL_CAP_APPEND, "append", + {offsetof(IcebergMetaEngine, append), + offsetof(IcebergMetaEngine, commit_append)}, 2}, + {DL_CAP_UPDATE, "update", + {offsetof(IcebergMetaEngine, update), + offsetof(IcebergMetaEngine, commit_update)}, 2}, + {DL_CAP_GET_FRAGMENT, "get fragment", + {offsetof(IcebergMetaEngine, get_fragment), + offsetof(IcebergMetaEngine, fragment_iter_next_batch), + offsetof(IcebergMetaEngine, fragment_iter_close)}, 3}, + {DL_CAP_REWRITE, "rewrite", + {offsetof(IcebergMetaEngine, plan_file_groups), + offsetof(IcebergMetaEngine, commit_file_groups), + offsetof(IcebergMetaEngine, file_group_iter_next), + offsetof(IcebergMetaEngine, file_group_iter_close)}, 4}, + {DL_CAP_ALTER, "alter", + {offsetof(IcebergMetaEngine, alter_table)}, 1}, + {DL_CAP_TRUNCATE, "truncate", + {offsetof(IcebergMetaEngine, truncate_table)}, 1} +}; + +/* + * Is the method at this offset present, and set? + * + * A method that falls beyond the engine's struct_size is not part of the + * object at all: reading it would run past what the engine allocated. Such a + * method counts as absent, which is exactly what prefix compatibility means -- + * an engine built against an older header stays loadable, and every capability + * whose family reaches into the missing tail must be left unset. + */ +static bool +meta_engine_method_is_nonnull(const IcebergMetaEngine *engine, size_t offset) +{ + void (*method)(void); + + if (offset + sizeof(method) > engine->struct_size) + return false; + + memcpy(&method, (const char *) engine + offset, sizeof(method)); + return method != NULL; +} + +DlErrCode +RegisterMetaEngine(const IcebergMetaEngine *engine) +{ + int i; + int j; + + if (engine == NULL) + return DL_ERR_INVALID_OPTION; + if (engine->abi_version != DL_META_ENGINE_ABI_VERSION) + return DL_ERR_INVALID_OPTION; + if (engine->struct_size < DL_META_ENGINE_V1_MIN_SIZE) + return DL_ERR_INVALID_OPTION; + if (engine->name == NULL) + return DL_ERR_INVALID_OPTION; + + for (i = 0; i < n_engines; i++) + { + if (strcmp(engines[i]->name, engine->name) == 0) + return DL_ERR_ALREADY_EXISTS; + } + if (n_engines >= lengthof(engines)) + return DL_ERR_INTERNAL; + + for (i = 0; i < lengthof(capability_families); i++) + { + const DlCapabilityFamily *family = &capability_families[i]; + bool capability_set = + (engine->capabilities & family->bit) != 0; + + for (j = 0; j < family->n; j++) + { + bool method_is_nonnull = + meta_engine_method_is_nonnull(engine, family->offsets[j]); + + if (capability_set != method_is_nonnull) + { + elog(WARNING, + "datalake_fdw: meta engine \"%s\" has an invalid %s capability family", + engine->name, family->family); + return DL_ERR_INVALID_OPTION; + } + } + } + + /* + * One engine per build, by design: nothing selects between implementations + * at run time, so a second registration would silently decide which one + * every table goes through, depending on registration order. + */ + if (n_engines > 0) + return DL_ERR_ALREADY_EXISTS; + + engines[n_engines++] = engine; + return DL_OK; +} + +const IcebergMetaEngine * +get_meta_engine(void) +{ + /* + * Exactly one engine is registered, by DatalakeRegisterMetaEngines(); the + * array exists so that adding a second implementation later is a matter of + * changing what gets registered, not of reworking the call sites. + */ + if (n_engines != 1) + elog(ERROR, + "datalake_fdw: expected exactly one metadata engine, found %d", + n_engines); + + return engines[0]; +} + +/* + * What every dispatch wrapper does before reaching the engine: discard detail + * recorded by an earlier call, so that a later report can only describe this + * one, and refuse a method the engine does not advertise. + * + * The reset happens before the capability check on purpose. A refusal produced + * here has no detail of its own, and leaving an earlier one in place would let + * it be reported as the cause. + */ +static DlErrCode +meta_engine_enter(const IcebergMetaEngine *engine, uint64 capability) +{ + dl_error_reset(); + + if (engine == NULL) + return DL_ERR_INTERNAL; + if ((engine->capabilities & capability) == 0) + return DL_ERR_NOT_SUPPORTED; + + return DL_OK; +} + +DlErrCode +meta_engine_create_table(const IcebergMetaEngine *engine, const MetaCtx *ctx, + const MetaTableDef *def, MetaTable **out) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->create_table(ctx, def, out); +} + +DlErrCode +meta_engine_drop_table(const IcebergMetaEngine *engine, const MetaCtx *ctx, + bool purge) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->drop_table(ctx, purge); +} + +DlErrCode +meta_engine_table_exists(const IcebergMetaEngine *engine, const MetaCtx *ctx, + bool *exists) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->table_exists(ctx, exists); +} + +DlErrCode +meta_engine_load_table(const IcebergMetaEngine *engine, const MetaCtx *ctx, + MetaTable **out) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->load_table(ctx, out); +} diff --git a/contrib/datalake_fdw/test/automation/Makefile b/contrib/datalake_fdw/test/automation/Makefile new file mode 100644 index 00000000000..fcedd800c0b --- /dev/null +++ b/contrib/datalake_fdw/test/automation/Makefile @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/test/automation/Makefile + +.PHONY: all test smoke-test check-services list-categories help clean + +all: test + +help: + @echo 'datalake_fdw automation tests' + @echo + @echo ' make test run the smoke categories (default)' + @echo ' make smoke-test same' + @echo ' make check-services report which external services answer' + @echo ' make list-categories list categories and what each one needs' + @echo ' make clean remove run output' + @echo + @echo ' CATEGORIES="..." run only these categories' + @echo + @echo 'Service addresses come from config/test_config.env, which reads them' + @echo 'from the environment first. A category whose services are absent is' + @echo 'skipped and reported as skipped.' + +test: smoke-test + +smoke-test: + @bash scripts/test/run_smoke_tests.sh + +check-services: + @bash scripts/setup/check_services.sh + +list-categories: + @printf '%-16s %s\n' 'CATEGORY' 'REQUIRES' + @printf '%-16s %s\n' 'iceberg_am' '(nothing external)' + +clean: + rm -rf sqlrepo/smoke/*/results diff --git a/contrib/datalake_fdw/test/automation/README.md b/contrib/datalake_fdw/test/automation/README.md new file mode 100644 index 00000000000..f6586ed0e5c --- /dev/null +++ b/contrib/datalake_fdw/test/automation/README.md @@ -0,0 +1,80 @@ + + +# datalake_fdw automation tests + +Lake tables are only half local. Once the metadata engine is connected, the +behaviour worth testing is what happens against a real Hive metastore, real +object storage and a real HDFS -- which comparison against a recorded transcript +cannot express, because the interesting cases are the ones where an external +service is slow, absent, or disagrees. This directory is where those tests go, +and it exists now so that they are added here rather than somewhere new. + +Everything currently here needs no external service. + +## Running + +```sh +make test # run the smoke categories +make check-services # what answers right now +make list-categories # categories, and what each one needs +``` + +Service addresses come from `config/test_config.env`, which takes them from the +environment first, so a run against an existing deployment needs no edit: + +```sh +DL_HMS_HOST=metastore.example DL_S3_ENDPOINT=http://minio:9000 make test +``` + +A category whose services are absent is **skipped and reported as skipped**, so +a developer without a metastore still gets a useful run and nobody reads a skip +as a pass. + +## Layout + +``` +config/ service addresses and switches +scripts/setup/ service probes +scripts/test/ category runners +scripts/utils/ shared shell helpers +sqlrepo/smoke/ one directory per category + iceberg_am/ DDL, refusals and privileges -- no external service +``` + +`sqlrepo/smoke/iceberg_am` holds the cases pg_regress runs; the module's +`Makefile` points `--inputdir` here, so `make installcheck` from the module +directory and `make test` from this one run the same cases. They live here +rather than in a `sql/` directory of their own so that there is one place to look +for test material. + +## What arrives with the metadata engine + +Named here so the shape is known before the code lands, rather than reserved as +empty directories: + +- `docker/` -- compose definitions bringing up a metastore, MinIO and a + single-node HDFS, so a category can run anywhere +- `prepare/` -- fixture loading per service: create the warehouse, the bucket, + the namespaces +- `lib/` -- SQL and shell fragments shared by categories +- `tools/` -- data generators for the volume and scale categories +- `sqlrepo/smoke/iceberg_hive`, `iceberg_s3`, `iceberg_hdfs` -- the read and + write paths against each service +- `sqlrepo/feature`, `sqlrepo/negative` -- everything past the smoke level diff --git a/contrib/datalake_fdw/test/automation/config/test_config.env b/contrib/datalake_fdw/test/automation/config/test_config.env new file mode 100644 index 00000000000..1389becc489 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/config/test_config.env @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Where the external services live. Overridden from the environment, so a run +# against an existing deployment needs no edit here. +# +# Nothing in the smoke suite needs any of these yet: the categories that do +# arrive with the metadata engine. They are named now because the runner +# already decides what to skip from them. + +# Hive metastore, for catalog type "hive". +DL_HMS_HOST="${DL_HMS_HOST:-localhost}" +DL_HMS_PORT="${DL_HMS_PORT:-9083}" + +# S3-compatible object storage, for volumes with an s3:// base path. +DL_S3_ENDPOINT="${DL_S3_ENDPOINT:-http://localhost:9000}" +DL_S3_BUCKET="${DL_S3_BUCKET:-datalake-test}" +DL_S3_REGION="${DL_S3_REGION:-us-east-1}" + +# HDFS namenode, for volumes with an hdfs:// base path. +DL_HDFS_HOST="${DL_HDFS_HOST:-localhost}" +DL_HDFS_PORT="${DL_HDFS_PORT:-8020}" + +# How long a service probe waits before calling the service absent. +DL_PROBE_TIMEOUT="${DL_PROBE_TIMEOUT:-3}" diff --git a/contrib/datalake_fdw/test/automation/scripts/setup/check_services.sh b/contrib/datalake_fdw/test/automation/scripts/setup/check_services.sh new file mode 100755 index 00000000000..dc300d9dc45 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/scripts/setup/check_services.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Report which external services are reachable. +# +# Informational on purpose: it exits 0 whether or not anything answered, because +# its output decides which categories the runner skips, and a missing service is +# a reason to skip a category rather than to fail a run. + +set -u + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../utils/common_functions.sh +. "$script_dir/../utils/common_functions.sh" + +dl_load_config + +report() +{ + local name="$1" where="$2" state="$3" + + printf '%-16s %-32s %s\n' "$name" "$where" "$state" +} + +printf '%-16s %-32s %s\n' 'SERVICE' 'ADDRESS' 'STATE' + +if dl_tcp_is_open "$DL_HMS_HOST" "$DL_HMS_PORT" "$DL_PROBE_TIMEOUT"; then + report 'hive-metastore' "$DL_HMS_HOST:$DL_HMS_PORT" 'available' +else + report 'hive-metastore' "$DL_HMS_HOST:$DL_HMS_PORT" 'absent' +fi + +if dl_http_is_open "$DL_S3_ENDPOINT" "$DL_PROBE_TIMEOUT"; then + report 's3' "$DL_S3_ENDPOINT" 'available' +else + report 's3' "$DL_S3_ENDPOINT" 'absent' +fi + +if dl_tcp_is_open "$DL_HDFS_HOST" "$DL_HDFS_PORT" "$DL_PROBE_TIMEOUT"; then + report 'hdfs' "$DL_HDFS_HOST:$DL_HDFS_PORT" 'available' +else + report 'hdfs' "$DL_HDFS_HOST:$DL_HDFS_PORT" 'absent' +fi + +exit 0 diff --git a/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh b/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh new file mode 100755 index 00000000000..ce707a1fed3 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Run the smoke categories under sqlrepo/smoke. +# +# A category names its required services in REQUIRED_SERVICES below. One with +# none is always run; one whose services are absent is skipped and reported as +# skipped, so a developer without a Hive metastore still gets a useful run and +# nobody mistakes a skip for a pass. + +set -u -o pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../utils/common_functions.sh +. "$script_dir/../utils/common_functions.sh" + +automation_dir="$(dl_automation_dir)" +module_dir="$(cd -- "$automation_dir/../.." && pwd)" + +dl_load_config + +# category:services -- an empty service list means "no external dependency" +CATEGORY_SERVICES=" +iceberg_am: +" + +categories="${CATEGORIES:-iceberg_am}" + +services_for() +{ + local category="$1" line + + while read -r line; do + [ -n "$line" ] || continue + case "$line" in + "$category":*) printf '%s' "${line#*:}"; return 0 ;; + esac + done <<-EOF + $CATEGORY_SERVICES + EOF + + printf 'unknown' +} + +service_is_available() +{ + case "$1" in + hive-metastore) + dl_tcp_is_open "$DL_HMS_HOST" "$DL_HMS_PORT" "$DL_PROBE_TIMEOUT" ;; + s3) + dl_http_is_open "$DL_S3_ENDPOINT" "$DL_PROBE_TIMEOUT" ;; + hdfs) + dl_tcp_is_open "$DL_HDFS_HOST" "$DL_HDFS_PORT" "$DL_PROBE_TIMEOUT" ;; + *) + return 1 ;; + esac +} + +run_iceberg_am() +{ + # These cases are expected-output cases, so pg_regress runs them; the module + # Makefile already points it at sqlrepo/smoke/iceberg_am. + make -C "$module_dir" USE_PGXS=1 installcheck +} + +failed=0 +skipped=0 +ran=0 + +for category in $categories; do + required="$(services_for "$category")" + + if [ "$required" = 'unknown' ]; then + dl_warn "unknown category \"$category\"" + failed=$((failed + 1)) + continue + fi + + missing='' + for service in $required; do + service_is_available "$service" || missing="$missing $service" + done + + if [ -n "$missing" ]; then + dl_info "SKIP $category (absent:$missing)" + skipped=$((skipped + 1)) + continue + fi + + dl_info "RUN $category" + case "$category" in + iceberg_am) run_iceberg_am ;; + *) dl_warn "category \"$category\" has no runner"; false ;; + esac + + if [ $? -eq 0 ]; then + ran=$((ran + 1)) + else + dl_warn "FAIL $category" + failed=$((failed + 1)) + fi +done + +dl_info "passed=$ran skipped=$skipped failed=$failed" +[ "$failed" -eq 0 ] diff --git a/contrib/datalake_fdw/test/automation/scripts/utils/common_functions.sh b/contrib/datalake_fdw/test/automation/scripts/utils/common_functions.sh new file mode 100644 index 00000000000..3a9d92298e1 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/scripts/utils/common_functions.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Shared helpers. Sourced, never executed. + +# Absolute path of the automation directory, whichever directory the caller +# started from. +dl_automation_dir() +{ + local here + here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + cd -- "$here/../.." && pwd +} + +dl_load_config() +{ + local automation_dir + automation_dir="$(dl_automation_dir)" + + # shellcheck source=../../config/test_config.env + . "$automation_dir/config/test_config.env" +} + +dl_info() +{ + printf '[automation] %s\n' "$*" +} + +dl_warn() +{ + printf '[automation] %s\n' "$*" >&2 +} + +dl_die() +{ + dl_warn "$*" + exit 1 +} + +# Name of a working timeout command, or empty when there is none. macOS ships +# neither; coreutils installs it as gtimeout. +dl_timeout_command() +{ + local candidate + + for candidate in timeout gtimeout; do + if command -v "$candidate" >/dev/null 2>&1; then + printf '%s' "$candidate" + return 0 + fi + done + + return 1 +} + +# True when something is listening on host:port. Uses bash's own /dev/tcp so +# that a probe needs no tool that might not be installed. +# +# Host and port are passed as arguments rather than interpolated into the +# program text: they come from the environment, and a shell metacharacter in one +# would otherwise be executed. Being unable to probe is reported as a harness +# failure, not as "service absent" -- silently skipping a category because a +# tool is missing is how a suite stops testing anything without saying so. +dl_tcp_is_open() +{ + local host="$1" port="$2" seconds="${3:-3}" timeout_cmd + + if ! timeout_cmd="$(dl_timeout_command)"; then + dl_die "no timeout command found (install coreutils for gtimeout)" + fi + + "$timeout_cmd" "$seconds" bash -c \ + 'exec 3<>/dev/tcp/"$1"/"$2"' _ "$host" "$port" 2>/dev/null +} + +# True when an HTTP endpoint answers at all; any status counts, because a probe +# asks whether the service is there, not whether a request would succeed. +dl_http_is_open() +{ + local url="$1" timeout="${2:-3}" + + command -v curl >/dev/null 2>&1 || return 1 + curl --silent --show-error --output /dev/null \ + --max-time "$timeout" "$url" 2>/dev/null +} diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_acl.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_acl.out new file mode 100644 index 00000000000..c03534b1ca3 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_acl.out @@ -0,0 +1,75 @@ +-- Foreign-server USAGE is required; user mappings remain optional. +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; +CREATE SERVER dlskel_acl_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_acl_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/acl', + endpoint 'http://fake:9000'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_user LOGIN; +RESET client_min_messages; +CREATE SCHEMA dlskel_s; +GRANT CREATE, USAGE ON SCHEMA dlskel_s TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); +ERROR: permission denied for foreign server dlskel_acl_cat +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_cat TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); +ERROR: permission denied for foreign server dlskel_acl_vol +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_vol TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); +NOTICE: stub engine: created iceberg table "dlskel_s.t" in catalog "dlskel_acl_cat" +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (username 'u', auth_method 'simple'); +-- AWS temporary credentials are three values; the mapping has to be able to +-- hold all of them. +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_vol + OPTIONS (access_key_id 'k', secret_access_key 's', session_token 't'); +-- A server-side key is not a user mapping key. +ALTER USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (ADD warehouse 'x'); +ERROR: invalid iceberg catalog user mapping option "warehouse" +RESET ROLE; +DROP SCHEMA dlskel_s CASCADE; +NOTICE: drop cascades to table dlskel_s.t +NOTICE: stub engine: dropped iceberg table "dlskel_s.t" from catalog "dlskel_acl_cat", keeping data +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_cat; +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_vol; +DROP SERVER dlskel_acl_cat; +DROP SERVER dlskel_acl_vol; +DROP ROLE dlskel_user; +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_ddl.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_ddl.out new file mode 100644 index 00000000000..eb61b4a08b0 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_ddl.out @@ -0,0 +1,227 @@ +-- Happy-path DDL, binding persistence, distributed catalog state, and drops. +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_cat_rest + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443'); +DROP SERVER dlskel_cat_rest; +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/prefix', + endpoint 'http://fake:9000'); +CREATE TABLE dlskel_t (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_t" in catalog "dlskel_cat" +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t'; + relname | amname | reloptions +----------+---------+---------------------------------------- + dlskel_t | iceberg | {catalog=dlskel_cat,volume=dlskel_vol} +(1 row) + +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t'::regclass; + policytype | distkey +------------+--------- + p | +(1 row) + +-- Exactly one pg_class row on each primary segment. Counting rows in total +-- would accept one segment missing its row as long as another had two, which is +-- the very divergence this is here to catch; so compare the set of segments that +-- have exactly one row against the set of primaries. +SELECT count(*) = 0 AS every_segment_has_exactly_one +FROM (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_class') + WHERE relname = 'dlskel_t' + GROUP BY gp_segment_id HAVING count(*) = 1) missing_or_duplicated; + every_segment_has_exactly_one +------------------------------- + t +(1 row) + +SELECT oid AS dlskel_cat_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_cat' +\gset +SELECT oid AS dlskel_vol_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_vol' +\gset +SELECT 'dlskel_t'::regclass::oid AS dlskel_t_oid +\gset +SELECT b.binding, + NOT EXISTS (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid + GROUP BY gp_segment_id HAVING count(*) = 1) + AS every_segment_has_exactly_one +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + binding | every_segment_has_exactly_one +---------+------------------------------- + catalog | t + volume | t +(2 rows) + +ANALYZE dlskel_t; +VACUUM dlskel_t; +SELECT reltuples IN (-1, 0) AS no_local_stats +FROM pg_class +WHERE oid = 'dlskel_t'::regclass; + no_local_stats +---------------- + t +(1 row) + +-- pg_dump writes DISTRIBUTED RANDOMLY into the CREATE TABLE it emits, so this +-- is the statement a restore replays; refusing it would mean refusing to +-- restore a dump this module produced. It has to yield the same policy as the +-- clause the module injects on its own. +CREATE TABLE dlskel_t_dump (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED RANDOMLY; +NOTICE: stub engine: created iceberg table "public.dlskel_t_dump" in catalog "dlskel_cat" +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t_dump'::regclass; + policytype | distkey +------------+--------- + p | +(1 row) + +DROP TABLE dlskel_t_dump; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t_dump" from catalog "dlskel_cat", keeping data +-- Dropping the table drops this database's reference to it; the lake data stays +-- unless the table said otherwise. The default and the explicit form both have +-- to be observable, which is why the stub reports which one it was asked for. +CREATE TABLE dlskel_t_keep (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_t_keep" in catalog "dlskel_cat" +DROP TABLE dlskel_t_keep; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t_keep" from catalog "dlskel_cat", keeping data +CREATE TABLE dlskel_t_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = true); +NOTICE: stub engine: created iceberg table "public.dlskel_t_purge" in catalog "dlskel_cat" +SELECT reloptions FROM pg_class WHERE relname = 'dlskel_t_purge'; + reloptions +----------------------------------------------------------- + {catalog=dlskel_cat,volume=dlskel_vol,purge_on_drop=true} +(1 row) + +DROP TABLE dlskel_t_purge; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t_purge" from catalog "dlskel_cat", purging data +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t2 (a int) USING iceberg; +NOTICE: stub engine: created iceberg table "public.dlskel_t2" in catalog "dlskel_cat" +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t2'; + relname | amname | reloptions +-----------+---------+---------------------------------------- + dlskel_t2 | iceberg | {catalog=dlskel_cat,volume=dlskel_vol} +(1 row) + +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +SET default_table_access_method = iceberg; +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t3 (a int); +NOTICE: stub engine: created iceberg table "public.dlskel_t3" in catalog "dlskel_cat" +\set HIDE_TABLEAM off +\d+ dlskel_t3 + Table "public.dlskel_t3" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+---------+---------+--------------+------------- + a | integer | | | | plain | | +Distributed randomly +Access method: iceberg +Options: catalog=dlskel_cat, volume=dlskel_vol + +RESET default_table_access_method; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +DROP SERVER dlskel_cat; +ERROR: cannot drop server dlskel_cat because other objects depend on it +DETAIL: table dlskel_t depends on server dlskel_cat +table dlskel_t2 depends on server dlskel_cat +table dlskel_t3 depends on server dlskel_cat +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP SERVER dlskel_vol; +ERROR: cannot drop server dlskel_vol because other objects depend on it +DETAIL: table dlskel_t depends on server dlskel_vol +table dlskel_t2 depends on server dlskel_vol +table dlskel_t3 depends on server dlskel_vol +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP TABLE dlskel_t; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t" from catalog "dlskel_cat", keeping data +SELECT b.binding, + NOT EXISTS (SELECT 1 FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid) + AS gone_from_every_segment +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + binding | gone_from_every_segment +---------+------------------------- + catalog | t + volume | t +(2 rows) + +DROP SERVER dlskel_cat CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table dlskel_t2 +drop cascades to table dlskel_t3 +NOTICE: stub engine: dropped iceberg table "public.dlskel_t3" from catalog "dlskel_cat", keeping data +NOTICE: stub engine: dropped iceberg table "public.dlskel_t2" from catalog "dlskel_cat", keeping data +DROP SERVER dlskel_vol CASCADE; +SELECT gp_segment_id, relname +FROM gp_dist_random('pg_class') +WHERE relname LIKE 'dlskel\_%' ESCAPE '\' +ORDER BY 1, 2; + gp_segment_id | relname +---------------+--------- +(0 rows) + +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out new file mode 100644 index 00000000000..db5fde07cf4 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out @@ -0,0 +1,457 @@ +-- Unsupported data paths, CREATE/ALTER guards, and binding validators. +-- Errors raised on a segment carry its address and pid, which vary per run. +-- start_matchsubs +-- m/ \(seg[0-9]+[^)]* pid=[0-9]+\)/ +-- s/ \(seg[0-9]+[^)]* pid=[0-9]+\)// +-- end_matchsubs +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages; +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/reject', + endpoint 'http://fake:9000'); +CREATE SERVER dlskel_free + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://free:9083'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_role; +RESET client_min_messages; +CREATE TYPE dlskel_type AS (a int); +CREATE TABLE dlskel_r (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_r" in catalog "dlskel_cat" +SELECT * FROM dlskel_r; +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +INSERT INTO dlskel_r VALUES (1, 'x'); +ERROR: iceberg: INSERT is not supported yet (seg1 172.17.0.2:40001 pid=92002) +UPDATE dlskel_r SET a = 1; +ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.2:40000 pid=92001) +DELETE FROM dlskel_r; +ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.2:40000 pid=92001) +COPY dlskel_r FROM stdin; +ERROR: iceberg: INSERT is not supported yet +CONTEXT: COPY dlskel_r, line 1 +COPY dlskel_r TO stdout; +ERROR: iceberg: SELECT is not supported yet +CREATE INDEX ON dlskel_r (a); +ERROR: iceberg: CREATE INDEX is not supported yet +SELECT * FROM dlskel_r TABLESAMPLE BERNOULLI (10); +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +-- A table from an earlier transaction takes the new-filelocator path rather +-- than the access method's truncate callback, so this is the case that would +-- silently report success if only the callback rejected it. +TRUNCATE dlskel_r; +ERROR: iceberg: TRUNCATE is not supported yet +-- Same for a table created in this transaction, which does reach the callback. +BEGIN; +CREATE TABLE dlskel_r_new (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_r_new" in catalog "dlskel_cat" +TRUNCATE dlskel_r_new; +ERROR: iceberg: TRUNCATE is not supported yet +ROLLBACK; +-- A multi-table TRUNCATE must refuse before truncating the heap beside it, so +-- the row below has to survive the attempt. +CREATE TABLE dlskel_heap (a int) DISTRIBUTED BY (a); +INSERT INTO dlskel_heap VALUES (1); +TRUNCATE dlskel_heap, dlskel_r; +ERROR: iceberg: TRUNCATE is not supported yet +SELECT count(*) AS heap_rows_kept FROM dlskel_heap; + heap_rows_kept +---------------- + 1 +(1 row) + +VACUUM FULL dlskel_r; +ERROR: iceberg: VACUUM FULL on iceberg tables is not supported yet +SELECT * FROM dlskel_r FOR UPDATE; +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +CREATE TABLE dlskel_bad_dist (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED BY (a); +ERROR: iceberg: DISTRIBUTED BY is not supported yet +CREATE TABLE dlskel_bad_part (a int) + PARTITION BY RANGE (a) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: partitioned tables is not supported yet +CREATE TABLE dlskel_bad_inherits (b text) + INHERITS (dlskel_r) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: INHERITS is not supported yet +CREATE TABLE dlskel_bad_typed OF dlskel_type + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: typed tables (OF type) is not supported yet +CREATE TEMP TABLE dlskel_bad_temp (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: TEMP tables is not supported yet +CREATE UNLOGGED TABLE dlskel_bad_unlogged (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: UNLOGGED tables is not supported yet +BEGIN; +CREATE TABLE dlskel_bad_oncommit (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + ON COMMIT DROP; +ERROR: iceberg: ON COMMIT is not supported yet +ROLLBACK; +CREATE TABLE dlskel_bad_tablespace (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + TABLESPACE pg_default; +ERROR: iceberg: TABLESPACE is not supported yet +CREATE TABLE dlskel_bad USING iceberg AS SELECT 1; +ERROR: iceberg: CREATE TABLE AS / CREATE MATERIALIZED VIEW is not supported yet +CREATE MATERIALIZED VIEW dlskel_mv USING iceberg AS SELECT 1; +ERROR: iceberg: CREATE TABLE AS / CREATE MATERIALIZED VIEW is not supported yet +-- Converting a heap into a lake table has to be refused too: the relation is +-- still a heap when the statement arrives, so the guard above does not see it. +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +ERROR: iceberg: ALTER TABLE ... SET ACCESS METHOD iceberg is not supported yet +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +ERROR: iceberg: ALTER TABLE ... SET ACCESS METHOD iceberg is not supported yet +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +-- A column-bearing clause still cannot be honoured, and neither can a +-- replicated policy. +CREATE TABLE dlskel_bad_repl (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED REPLICATED; +ERROR: iceberg: DISTRIBUTED REPLICATED is not supported yet +-- Renaming a schema would repoint its lake tables at a different external +-- namespace, so a schema holding one is refused -- and, just as importantly, a +-- schema holding none is not: the guard has to be no wider than the problem. +CREATE SCHEMA dlskel_sch; +CREATE TABLE dlskel_sch.t (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "dlskel_sch.t" in catalog "dlskel_cat" +BEGIN; +ALTER SCHEMA dlskel_sch RENAME TO dlskel_sch2; +ERROR: iceberg: RENAME on schemas containing iceberg tables is not supported yet +ROLLBACK; +CREATE SCHEMA dlskel_plain; +ALTER SCHEMA dlskel_plain RENAME TO dlskel_plain2; +DROP SCHEMA dlskel_plain2; +-- Renaming a wrapper breaks every mapping lookup at once, including the one +-- DROP needs, so it is refused whether or not a table exists yet. +-- Each attempt is rolled back: if the guard ever regresses, an accepted rename +-- would leave the extension's own wrapper under a different name, which nothing +-- in the cleanup below can undo and which breaks every later run. +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_catalog_fdw RENAME TO dlskel_other_fdw; +ERROR: iceberg: RENAME on the iceberg foreign-data wrappers is not supported yet +ROLLBACK; +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_volume_fdw RENAME TO dlskel_other_fdw; +ERROR: iceberg: RENAME on the iceberg foreign-data wrappers is not supported yet +ROLLBACK; +ALTER TABLE dlskel_r ADD COLUMN c int; +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET (fillfactor = 90); +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET ACCESS METHOD heap; +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET DISTRIBUTED BY (a); +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r RENAME TO dlskel_r2; +ERROR: iceberg: RENAME on iceberg tables is not supported yet +ALTER TABLE dlskel_r RENAME COLUMN a TO aa; +ERROR: iceberg: RENAME on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET SCHEMA public; +ERROR: iceberg: SET SCHEMA on iceberg tables is not supported yet +-- OWNER TO is the one ALTER TABLE form that goes through: pg_dump writes it for +-- every table, and ownership cannot reach the external table. Put it back +-- afterwards so the rest of the file still owns what it created. +ALTER TABLE dlskel_r OWNER TO dlskel_role; +SELECT relname, pg_get_userbyid(relowner) AS owner +FROM pg_class WHERE relname = 'dlskel_r'; + relname | owner +----------+------------- + dlskel_r | dlskel_role +(1 row) + +ALTER TABLE dlskel_r OWNER TO CURRENT_USER; +ALTER SERVER dlskel_cat + OPTIONS (SET uri 'thrift://other:9083'); +ERROR: iceberg: ALTER SERVER on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_cat VERSION '2'; +ERROR: iceberg: ALTER SERVER on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_cat RENAME TO dlskel_cat2; +ERROR: iceberg: RENAME on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_cat OWNER TO dlskel_role; +ERROR: iceberg: ALTER OWNER on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_free + OPTIONS (SET uri 'thrift://other:9083'); +ALTER SERVER dlskel_free VERSION '2'; +ALTER SERVER dlskel_free OWNER TO dlskel_role; +ALTER SERVER dlskel_free RENAME TO dlskel_free2; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +CREATE TABLE dlskel_bad_missing_server (a int) + USING iceberg + WITH (catalog = 'dlskel_missing', volume = 'dlskel_vol'); +ERROR: server "dlskel_missing" does not exist +CREATE TABLE dlskel_bad_wrong_catalog (a int) + USING iceberg + WITH (catalog = 'dlskel_vol', volume = 'dlskel_vol'); +ERROR: server "dlskel_vol" is not an iceberg catalog server +CREATE TABLE dlskel_bad_no_catalog (a int) + USING iceberg + WITH (volume = 'dlskel_vol'); +ERROR: no catalog specified +HINT: Specify WITH (catalog = '...') or SET iceberg.default_catalog. +CREATE TABLE dlskel_bad_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = 'perhaps'); +ERROR: invalid value for boolean option "purge_on_drop": perhaps +CREATE TABLE dlskel_bad_reloption (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', nonsense = 'x'); +ERROR: unrecognized parameter "nonsense" +CREATE SERVER dlskel_bad_server + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', nonsense 'x'); +ERROR: invalid iceberg catalog server option "nonsense" +HINT: Allowed options are "type", "uri", "catalog_name", "warehouse" and "polaris_server_realm". +CREATE SERVER dlskel_bad_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (secret_key 'x'); +ERROR: credential option "secret_key" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +-- The catalog wrapper has its own allowlist, and its own credential keys. +CREATE SERVER dlskel_bad_cat_secret + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', password 'x'); +ERROR: credential option "password" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_cat_token + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'polaris', uri 'https://fake:443', client_secret 'x'); +ERROR: credential option "client_secret" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +-- Every key the credential list mirrors from an option module, so that a key +-- renamed in one place and not the other fails here instead of silently +-- ceasing to be caught. +CREATE SERVER dlskel_bad_cat_user + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', username 'u'); +ERROR: credential option "username" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_cat_keytab + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', krb_client_keytab '/k'); +ERROR: credential option "krb_client_keytab" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_cat_clientid + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443', client_id 'i'); +ERROR: credential option "client_id" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_vol_token + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', session_token 't'); +ERROR: credential option "session_token" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_vol_keyid + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', access_key_id 'k'); +ERROR: credential option "access_key_id" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_vol_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', secret_access_key 's'); +ERROR: credential option "secret_access_key" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +-- Option names are matched exactly, as the server itself matches them; a quoted +-- variant is a different, unknown option rather than a second spelling. +CREATE SERVER dlskel_bad_cat_case + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS ("TYPE" 'hive', uri 'thrift://fake:9083'); +ERROR: invalid iceberg catalog server option "TYPE" +HINT: Allowed options are "type", "uri", "catalog_name", "warehouse" and "polaris_server_realm". +-- A catalog type outside the vocabulary, and one that is in it but has no +-- implementation behind it yet. +CREATE SERVER dlskel_bad_cat_type + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'nosuchcatalog', uri 'thrift://fake:9083'); +ERROR: invalid iceberg catalog type "nosuchcatalog" +HINT: Allowed types are "hive", "rest" and "builtin"; "polaris" is accepted as an alias of "rest". +CREATE SERVER dlskel_bad_cat_hadoop + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hadoop', uri 'hdfs://fake:8020'); +ERROR: iceberg: catalog type "hadoop" is not supported yet +-- builtin needs no url; supplying one means the two disagree. +CREATE SERVER dlskel_bad_cat_builtin + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'builtin', uri 'thrift://fake:9083'); +ERROR: iceberg catalog type "builtin" forbids server option "uri" +-- hive does need one. +CREATE SERVER dlskel_bad_cat_nourl + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive'); +ERROR: iceberg catalog type "hive" requires server option "uri" +-- The realm applies to one catalog type only. +CREATE SERVER dlskel_bad_cat_realm + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', + polaris_server_realm 'OTHER'); +ERROR: iceberg catalog server option "polaris_server_realm" applies only to catalog type "rest" +-- An empty value is refused where it is written, not where it is first read. +CREATE SERVER dlskel_bad_cat_empty + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri ''); +ERROR: iceberg catalog server option "uri" cannot be empty +-- Not a boolean. +CREATE SERVER dlskel_bad_vol_bool + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', path_style_access 'perhaps'); +ERROR: invalid boolean value "perhaps" for option "path_style_access" +CREATE SERVER dlskel_bad_nopath + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (endpoint 'http://fake:9000'); +ERROR: iceberg volume server option "base_path" is required +CREATE SERVER dlskel_bad_scheme + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'ftp://x/y'); +ERROR: invalid iceberg volume base_path "ftp://x/y" +DETAIL: location URI "ftp://x/y" has unsupported scheme; expected s3 or hdfs +CREATE SERVER dlskel_bad_authority + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://'); +ERROR: invalid iceberg volume base_path "s3://" +DETAIL: location URI "s3://" has an empty authority +CREATE SERVER dlskel_bad_query + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://b/p?versionId=3'); +ERROR: invalid iceberg volume base_path "s3://b/p?versionId=3" +DETAIL: location URI "s3://b/p?versionId=3" must not contain a query +CREATE SERVER dlskel_bad_userinfo + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://user@b/p'); +ERROR: invalid iceberg volume base_path "s3://user@b/p" +DETAIL: location URI "s3://user@b/p" must not contain userinfo +CREATE SERVER dlskel_bad_bucket + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://UPPER_case/p'); +ERROR: invalid iceberg volume base_path "s3://UPPER_case/p" +DETAIL: s3 bucket in location URI "s3://UPPER_case/p" must start and end with a lowercase letter or digit +-- The metadata engine is not selectable, so naming one is an unknown option. +CREATE TABLE dlskel_bad_engine (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + engine = 'agent'); +ERROR: unrecognized parameter "engine" +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_acl.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_acl.sql new file mode 100644 index 00000000000..40d2020a0ce --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_acl.sql @@ -0,0 +1,78 @@ +-- Foreign-server USAGE is required; user mappings remain optional. + +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; + +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; + +CREATE SERVER dlskel_acl_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_acl_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/acl', + endpoint 'http://fake:9000'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_user LOGIN; +RESET client_min_messages; +CREATE SCHEMA dlskel_s; +GRANT CREATE, USAGE ON SCHEMA dlskel_s TO dlskel_user; + +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); + +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_cat TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); + +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_vol TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); + +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (username 'u', auth_method 'simple'); +-- AWS temporary credentials are three values; the mapping has to be able to +-- hold all of them. +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_vol + OPTIONS (access_key_id 'k', secret_access_key 's', session_token 't'); +-- A server-side key is not a user mapping key. +ALTER USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (ADD warehouse 'x'); + +RESET ROLE; +DROP SCHEMA dlskel_s CASCADE; +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_cat; +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_vol; +DROP SERVER dlskel_acl_cat; +DROP SERVER dlskel_acl_vol; +DROP ROLE dlskel_user; + +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_ddl.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_ddl.sql new file mode 100644 index 00000000000..365fe7e0b00 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_ddl.sql @@ -0,0 +1,162 @@ +-- Happy-path DDL, binding persistence, distributed catalog state, and drops. + +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; + +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; + +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_cat_rest + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443'); +DROP SERVER dlskel_cat_rest; +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/prefix', + endpoint 'http://fake:9000'); + +CREATE TABLE dlskel_t (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); + +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t'; + +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t'::regclass; + +-- Exactly one pg_class row on each primary segment. Counting rows in total +-- would accept one segment missing its row as long as another had two, which is +-- the very divergence this is here to catch; so compare the set of segments that +-- have exactly one row against the set of primaries. +SELECT count(*) = 0 AS every_segment_has_exactly_one +FROM (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_class') + WHERE relname = 'dlskel_t' + GROUP BY gp_segment_id HAVING count(*) = 1) missing_or_duplicated; + +SELECT oid AS dlskel_cat_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_cat' +\gset +SELECT oid AS dlskel_vol_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_vol' +\gset +SELECT 'dlskel_t'::regclass::oid AS dlskel_t_oid +\gset + +SELECT b.binding, + NOT EXISTS (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid + GROUP BY gp_segment_id HAVING count(*) = 1) + AS every_segment_has_exactly_one +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + +ANALYZE dlskel_t; +VACUUM dlskel_t; +SELECT reltuples IN (-1, 0) AS no_local_stats +FROM pg_class +WHERE oid = 'dlskel_t'::regclass; + +-- pg_dump writes DISTRIBUTED RANDOMLY into the CREATE TABLE it emits, so this +-- is the statement a restore replays; refusing it would mean refusing to +-- restore a dump this module produced. It has to yield the same policy as the +-- clause the module injects on its own. +CREATE TABLE dlskel_t_dump (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED RANDOMLY; +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t_dump'::regclass; +DROP TABLE dlskel_t_dump; + +-- Dropping the table drops this database's reference to it; the lake data stays +-- unless the table said otherwise. The default and the explicit form both have +-- to be observable, which is why the stub reports which one it was asked for. +CREATE TABLE dlskel_t_keep (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +DROP TABLE dlskel_t_keep; +CREATE TABLE dlskel_t_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = true); +SELECT reloptions FROM pg_class WHERE relname = 'dlskel_t_purge'; +DROP TABLE dlskel_t_purge; + +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t2 (a int) USING iceberg; +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t2'; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; + +SET default_table_access_method = iceberg; +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t3 (a int); +\set HIDE_TABLEAM off +\d+ dlskel_t3 +RESET default_table_access_method; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; + +DROP SERVER dlskel_cat; +DROP SERVER dlskel_vol; + +DROP TABLE dlskel_t; +SELECT b.binding, + NOT EXISTS (SELECT 1 FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid) + AS gone_from_every_segment +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + +DROP SERVER dlskel_cat CASCADE; +DROP SERVER dlskel_vol CASCADE; + +SELECT gp_segment_id, relname +FROM gp_dist_random('pg_class') +WHERE relname LIKE 'dlskel\_%' ESCAPE '\' +ORDER BY 1, 2; + +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql new file mode 100644 index 00000000000..6f376f61e49 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql @@ -0,0 +1,379 @@ +-- Unsupported data paths, CREATE/ALTER guards, and binding validators. +-- Errors raised on a segment carry its address and pid, which vary per run. +-- start_matchsubs +-- m/ \(seg[0-9]+[^)]* pid=[0-9]+\)/ +-- s/ \(seg[0-9]+[^)]* pid=[0-9]+\)// +-- end_matchsubs + +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages; + +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; + +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/reject', + endpoint 'http://fake:9000'); +CREATE SERVER dlskel_free + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://free:9083'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_role; +RESET client_min_messages; +CREATE TYPE dlskel_type AS (a int); + +CREATE TABLE dlskel_r (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); + +SELECT * FROM dlskel_r; +INSERT INTO dlskel_r VALUES (1, 'x'); +UPDATE dlskel_r SET a = 1; +DELETE FROM dlskel_r; +COPY dlskel_r FROM stdin; +1 x +\. +COPY dlskel_r TO stdout; +CREATE INDEX ON dlskel_r (a); +SELECT * FROM dlskel_r TABLESAMPLE BERNOULLI (10); + +-- A table from an earlier transaction takes the new-filelocator path rather +-- than the access method's truncate callback, so this is the case that would +-- silently report success if only the callback rejected it. +TRUNCATE dlskel_r; + +-- Same for a table created in this transaction, which does reach the callback. +BEGIN; +CREATE TABLE dlskel_r_new (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +TRUNCATE dlskel_r_new; +ROLLBACK; + +-- A multi-table TRUNCATE must refuse before truncating the heap beside it, so +-- the row below has to survive the attempt. +CREATE TABLE dlskel_heap (a int) DISTRIBUTED BY (a); +INSERT INTO dlskel_heap VALUES (1); +TRUNCATE dlskel_heap, dlskel_r; +SELECT count(*) AS heap_rows_kept FROM dlskel_heap; + +VACUUM FULL dlskel_r; +SELECT * FROM dlskel_r FOR UPDATE; + +CREATE TABLE dlskel_bad_dist (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED BY (a); +CREATE TABLE dlskel_bad_part (a int) + PARTITION BY RANGE (a) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_inherits (b text) + INHERITS (dlskel_r) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_typed OF dlskel_type + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE TEMP TABLE dlskel_bad_temp (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE UNLOGGED TABLE dlskel_bad_unlogged (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +BEGIN; +CREATE TABLE dlskel_bad_oncommit (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + ON COMMIT DROP; +ROLLBACK; +CREATE TABLE dlskel_bad_tablespace (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + TABLESPACE pg_default; + +CREATE TABLE dlskel_bad USING iceberg AS SELECT 1; +CREATE MATERIALIZED VIEW dlskel_mv USING iceberg AS SELECT 1; + +-- Converting a heap into a lake table has to be refused too: the relation is +-- still a heap when the statement arrives, so the guard above does not see it. +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; + +-- A column-bearing clause still cannot be honoured, and neither can a +-- replicated policy. +CREATE TABLE dlskel_bad_repl (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED REPLICATED; + +-- Renaming a schema would repoint its lake tables at a different external +-- namespace, so a schema holding one is refused -- and, just as importantly, a +-- schema holding none is not: the guard has to be no wider than the problem. +CREATE SCHEMA dlskel_sch; +CREATE TABLE dlskel_sch.t (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +BEGIN; +ALTER SCHEMA dlskel_sch RENAME TO dlskel_sch2; +ROLLBACK; +CREATE SCHEMA dlskel_plain; +ALTER SCHEMA dlskel_plain RENAME TO dlskel_plain2; +DROP SCHEMA dlskel_plain2; + +-- Renaming a wrapper breaks every mapping lookup at once, including the one +-- DROP needs, so it is refused whether or not a table exists yet. +-- Each attempt is rolled back: if the guard ever regresses, an accepted rename +-- would leave the extension's own wrapper under a different name, which nothing +-- in the cleanup below can undo and which breaks every later run. +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_catalog_fdw RENAME TO dlskel_other_fdw; +ROLLBACK; +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_volume_fdw RENAME TO dlskel_other_fdw; +ROLLBACK; + +ALTER TABLE dlskel_r ADD COLUMN c int; +ALTER TABLE dlskel_r SET (fillfactor = 90); +ALTER TABLE dlskel_r SET ACCESS METHOD heap; +ALTER TABLE dlskel_r SET DISTRIBUTED BY (a); +ALTER TABLE dlskel_r RENAME TO dlskel_r2; +ALTER TABLE dlskel_r RENAME COLUMN a TO aa; +ALTER TABLE dlskel_r SET SCHEMA public; +-- OWNER TO is the one ALTER TABLE form that goes through: pg_dump writes it for +-- every table, and ownership cannot reach the external table. Put it back +-- afterwards so the rest of the file still owns what it created. +ALTER TABLE dlskel_r OWNER TO dlskel_role; +SELECT relname, pg_get_userbyid(relowner) AS owner +FROM pg_class WHERE relname = 'dlskel_r'; +ALTER TABLE dlskel_r OWNER TO CURRENT_USER; + +ALTER SERVER dlskel_cat + OPTIONS (SET uri 'thrift://other:9083'); +ALTER SERVER dlskel_cat VERSION '2'; +ALTER SERVER dlskel_cat RENAME TO dlskel_cat2; +ALTER SERVER dlskel_cat OWNER TO dlskel_role; + +ALTER SERVER dlskel_free + OPTIONS (SET uri 'thrift://other:9083'); +ALTER SERVER dlskel_free VERSION '2'; +ALTER SERVER dlskel_free OWNER TO dlskel_role; +ALTER SERVER dlskel_free RENAME TO dlskel_free2; + +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +CREATE TABLE dlskel_bad_missing_server (a int) + USING iceberg + WITH (catalog = 'dlskel_missing', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_wrong_catalog (a int) + USING iceberg + WITH (catalog = 'dlskel_vol', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_no_catalog (a int) + USING iceberg + WITH (volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = 'perhaps'); +CREATE TABLE dlskel_bad_reloption (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', nonsense = 'x'); + +CREATE SERVER dlskel_bad_server + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', nonsense 'x'); +CREATE SERVER dlskel_bad_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (secret_key 'x'); +-- The catalog wrapper has its own allowlist, and its own credential keys. +CREATE SERVER dlskel_bad_cat_secret + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', password 'x'); +CREATE SERVER dlskel_bad_cat_token + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'polaris', uri 'https://fake:443', client_secret 'x'); +-- Every key the credential list mirrors from an option module, so that a key +-- renamed in one place and not the other fails here instead of silently +-- ceasing to be caught. +CREATE SERVER dlskel_bad_cat_user + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', username 'u'); +CREATE SERVER dlskel_bad_cat_keytab + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', krb_client_keytab '/k'); +CREATE SERVER dlskel_bad_cat_clientid + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443', client_id 'i'); +CREATE SERVER dlskel_bad_vol_token + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', session_token 't'); +CREATE SERVER dlskel_bad_vol_keyid + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', access_key_id 'k'); +CREATE SERVER dlskel_bad_vol_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', secret_access_key 's'); +-- Option names are matched exactly, as the server itself matches them; a quoted +-- variant is a different, unknown option rather than a second spelling. +CREATE SERVER dlskel_bad_cat_case + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS ("TYPE" 'hive', uri 'thrift://fake:9083'); +-- A catalog type outside the vocabulary, and one that is in it but has no +-- implementation behind it yet. +CREATE SERVER dlskel_bad_cat_type + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'nosuchcatalog', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_bad_cat_hadoop + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hadoop', uri 'hdfs://fake:8020'); +-- builtin needs no url; supplying one means the two disagree. +CREATE SERVER dlskel_bad_cat_builtin + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'builtin', uri 'thrift://fake:9083'); +-- hive does need one. +CREATE SERVER dlskel_bad_cat_nourl + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive'); +-- The realm applies to one catalog type only. +CREATE SERVER dlskel_bad_cat_realm + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', + polaris_server_realm 'OTHER'); +-- An empty value is refused where it is written, not where it is first read. +CREATE SERVER dlskel_bad_cat_empty + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri ''); +-- Not a boolean. +CREATE SERVER dlskel_bad_vol_bool + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', path_style_access 'perhaps'); +CREATE SERVER dlskel_bad_nopath + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (endpoint 'http://fake:9000'); +CREATE SERVER dlskel_bad_scheme + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'ftp://x/y'); +CREATE SERVER dlskel_bad_authority + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://'); +CREATE SERVER dlskel_bad_query + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://b/p?versionId=3'); +CREATE SERVER dlskel_bad_userinfo + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://user@b/p'); +CREATE SERVER dlskel_bad_bucket + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://UPPER_case/p'); + +-- The metadata engine is not selectable, so naming one is an unknown option. +CREATE TABLE dlskel_bad_engine (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + engine = 'agent'); + +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages;