[Draft] Master catalog spi review 22

.github/workflows/license-eyes.yml

@@ -74,19 +74,21 @@ jobs:
           if [ -z "$acmr" ]; then
             exit 0
           fi
-          {
-            echo "added_modified<<EOF"
-            echo "$acmr"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
+          # Hand the list over as a file, not as a step output consumed through an
+          # environment variable: a single env string is capped at MAX_ARG_STRLEN
+          # (128KiB), and a large PR blows past it, so exec'ing the next step's
+          # shell fails with "Argument list too long". Keep it out of the
+          # workspace, where the full-config fallback would license-check it.
+          printf '%s\n' "$acmr" > "${RUNNER_TEMP}/licenserc-changed-files.txt"
+          echo "changed_files_file=${RUNNER_TEMP}/licenserc-changed-files.txt" >> "$GITHUB_OUTPUT"
           echo "config_file=.licenserc-incremental.yaml" >> "$GITHUB_OUTPUT"
 
       - name: Generate incremental licenserc
         if: >-
           github.event_name == 'pull_request' &&
-          steps.changed-files.outputs.added_modified != ''
+          steps.changed-files.outputs.changed_files_file != ''
         env:
-          CHANGED_FILES: ${{ steps.changed-files.outputs.added_modified }}
+          CHANGED_FILES_FILE: ${{ steps.changed-files.outputs.changed_files_file }}
         run: |
           python3 - <<'EOF'
           import sys
@@ -97,8 +99,8 @@ jobs:
           with open('.licenserc.yaml') as f:
               config = yaml.safe_load(f)
 
-          changed = os.environ.get('CHANGED_FILES', '').strip().split('\n')
-          changed = [p.strip() for p in changed if p.strip()]
+          with open(os.environ['CHANGED_FILES_FILE']) as f:
+              changed = [p.strip() for p in f if p.strip()]
 
           config['header']['paths'] = changed
 

.github/workflows/third_party_review.yml

@@ -37,7 +37,13 @@ jobs:
           # ([String]). Only allow these licenses (optional)
           # Possible values: Any SPDX-compliant license identifiers or expressions from https://spdx.org/licenses/
           allow-licenses:  BSD-2-Clause, BSD-3-Clause, MIT, Apache-2.0, EPL-2.0, MPL-2.0, CC0-1.0
-          #allow-ghsas: GHSA-abcd-1234-5679, GHSA-efgh-1234-5679
+          # ([String]). Acknowledged advisories that must not fail the review (optional)
+          # org.codehaus.jackson:jackson-mapper-asl (GHSA-c27h-mcmw-48hv, GHSA-r6j9-8759-g62w):
+          # legacy Jackson 1.x is EOL and neither advisory has a fixed version. Hive's metastore
+          # JSONMessageDeserializer, which the HMS notification-event path builds on, runs on it, so
+          # these classes already ship inside the prebuilt org.apache.doris:hive-catalog-shade jar;
+          # shading them in-repo only makes the dependency visible to this scan.
+          allow-ghsas: GHSA-c27h-mcmw-48hv, GHSA-r6j9-8759-g62w
           # ([String]). Block pull requests that introduce vulnerabilities in the scopes that match this list (optional)
           # Possible values: "development", "runtime", "unknown"
           fail-on-scopes: development, runtime

.gitleaks.toml

@@ -6,6 +6,12 @@ useDefault = true
 [[rules]]
 id = "generic-api-key"
 
+[[rules.allowlists]]
+description = "Ignore prose 'version token: iceberg/paimon' in code comments/plan-doc — a cache version-token discussion, not a credential (generic-api-key false positive: 'iceberg/paimon' entropy 3.52 barely clears the 3.5 threshold)"
+condition = "AND"
+regexTarget = "match"
+regexes = ['''token:\s*iceberg/paimon''']
+
 [[rules.allowlists]]
 description = "Ignore the fake OIDC token fixture in AuthenticatorManagerTest"
 condition = "AND"

.licenserc.yaml

@@ -49,6 +49,16 @@ header:
     - "docs/.markdownlintignore"
     - "fe/fe-core/src/test/resources/data/net_snmp_normal"
     - "fe/fe-core/src/test/resources/stageUtilTest.txt"
+    # Recorded method-surface baseline; ConnectorMetadataSurfaceTest reads every
+    # non-blank line as a method signature, so a header would break the test.
+    - "fe/fe-connector/fe-connector-api/src/test/resources/connector-metadata-methods.txt"
+    # Recorded plugin API surface baselines, one per plugin family. Same reason as
+    # above: each *PluginSurfaceTest.readBaseline() adds every non-blank line to the
+    # expected signature set, so a header would be read back as phantom signatures.
+    - "fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt"
+    - "fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt"
+    - "fe/fe-authentication/fe-authentication-spi/src/test/resources/authentication-plugin-surface.txt"
+    - "fe/fe-core/src/test/resources/lineage-plugin-surface.txt"
     - "fe/fe-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java"
     - "fe/fe-core/src/main/antlr4/org/apache/doris/nereids/JavaLexer.g4"
     - "fe/fe-core/src/main/antlr4/org/apache/doris/nereids/JavaParser.g4"

be/src/common/config.cpp

@@ -1550,7 +1550,10 @@ DEFINE_mInt64(s3_put_token_limit, "0");
 DEFINE_mInt64(s3_rate_limiter_log_interval, "1000");
 DEFINE_Validator(s3_rate_limiter_log_interval, [](int64_t config) -> bool { return config >= 0; });
 
-DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors");
+// The dir TrinoConnectorPluginLoader loads Trino's own plugins from, used verbatim. Keep the default
+// in sync with FE Config.trino_connector_plugin_dir: FE and BE load the same plugins and an operator
+// who leaves both untouched expects both to find them.
+DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/trino_plugins");
 
 // ca_cert_file is in this path by default, Normally no modification is required
 // ca cert default path is different from different OS

be/src/exec/scan/meta_scanner.cpp

@@ -232,9 +232,6 @@ Status MetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) {
     VLOG_CRITICAL << "MetaScanner::_fetch_metadata";
     TFetchSchemaTableDataRequest request;
     switch (meta_scan_range.metadata_type) {
-    case TMetadataType::HUDI:
-        RETURN_IF_ERROR(_build_hudi_metadata_request(meta_scan_range, &request));
-        break;
     case TMetadataType::BACKENDS:
         RETURN_IF_ERROR(_build_backends_metadata_request(meta_scan_range, &request));
         break;
@@ -297,26 +294,6 @@ Status MetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) {
     return Status::OK();
 }
 
-Status MetaScanner::_build_hudi_metadata_request(const TMetaScanRange& meta_scan_range,
-                                                 TFetchSchemaTableDataRequest* request) {
-    VLOG_CRITICAL << "MetaScanner::_build_hudi_metadata_request";
-    if (!meta_scan_range.__isset.hudi_params) {
-        return Status::InternalError("Can not find THudiMetadataParams from meta_scan_range.");
-    }
-
-    // create request
-    request->__set_cluster_name("");
-    request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE);
-
-    // create TMetadataTableRequestParams
-    TMetadataTableRequestParams metadata_table_params;
-    metadata_table_params.__set_metadata_type(TMetadataType::HUDI);
-    metadata_table_params.__set_hudi_metadata_params(meta_scan_range.hudi_params);
-
-    request->__set_metada_table_params(metadata_table_params);
-    return Status::OK();
-}
-
 Status MetaScanner::_build_backends_metadata_request(const TMetaScanRange& meta_scan_range,
                                                      TFetchSchemaTableDataRequest* request) {
     VLOG_CRITICAL << "MetaScanner::_build_backends_metadata_request";

be/src/exec/scan/meta_scanner.h

@@ -61,8 +61,6 @@ class MetaScanner : public Scanner {
 private:
     Status _fill_block_with_remote_data(const std::vector<MutableColumnPtr>& columns);
     Status _fetch_metadata(const TMetaScanRange& meta_scan_range);
-    Status _build_hudi_metadata_request(const TMetaScanRange& meta_scan_range,
-                                        TFetchSchemaTableDataRequest* request);
     Status _build_backends_metadata_request(const TMetaScanRange& meta_scan_range,
                                             TFetchSchemaTableDataRequest* request);
     Status _build_frontends_metadata_request(const TMetaScanRange& meta_scan_range,

be/src/format/parquet/vparquet_reader.cpp

@@ -497,6 +497,19 @@ Status ParquetReader::_do_init_reader(ReaderInitContext* base_ctx) {
         _fill_missing_cols.clear();
         _fill_missing_defaults.clear();
         for (const auto& col_name : base_ctx->column_names) {
+            // A projected column that is not present at all in the table-side schema tree means the
+            // schema info from FE (e.g. the paimon/iceberg history_schema_info, or the hive/orc name
+            // map) is inconsistent with the scan projection. Fail this query loudly instead of aborting
+            // the whole BE process via the children_column_exists DCHECK (or an std::out_of_range from
+            // children.at() in a release build). A column that IS known but missing from the data file
+            // keeps its key (add_not_exist_children) and is correctly classified as fill-missing below.
+            if (!_table_info_node_ptr->has_children_column(col_name)) {
+                return Status::InternalError(
+                        "schema mapping is missing projected column '{}'; the schema info from FE "
+                        "is "
+                        "inconsistent with the scan projection (file: {})",
+                        col_name, _scan_range.path);
+            }
             if (!_table_info_node_ptr->children_column_exists(col_name)) {
                 _fill_missing_cols.insert(col_name);
             }

be/src/format/table/table_schema_change_helper.h

@@ -72,6 +72,16 @@ class TableSchemaChangeHelper {
                     "children_column_exists should not be called on base TableInfoNode");
         }
 
+        // Presence-only check (does NOT DCHECK). Distinct from children_column_exists, which asserts
+        // the key exists and then reports the file-side exists flag. Callers use this to reject a
+        // projected column that is absent from the table-side schema tree (an FE/BE schema-contract
+        // mismatch) BEFORE calling children_column_exists, turning a would-be process abort into a
+        // graceful per-query error.
+        virtual bool has_children_column(std::string table_column_name) const {
+            throw std::logic_error(
+                    "has_children_column should not be called on base TableInfoNode");
+        }
+
         virtual std::optional<InitialDefaultValue> children_initial_default_value(
                 std::string) const {
             return std::nullopt;
@@ -120,6 +130,8 @@ class TableSchemaChangeHelper {
 
         bool children_column_exists(std::string table_column_name) const override { return true; }
 
+        bool has_children_column(std::string table_column_name) const override { return true; }
+
         std::shared_ptr<Node> get_element_node() const override { return get_instance(); }
 
         std::shared_ptr<Node> get_key_node() const override { return get_instance(); }
@@ -180,6 +192,10 @@ class TableSchemaChangeHelper {
             return children.at(table_column_name).exists;
         }
 
+        bool has_children_column(std::string table_column_name) const override {
+            return children.contains(table_column_name);
+        }
+
         std::optional<InitialDefaultValue> children_initial_default_value(
                 std::string table_column_name) const override {
             DCHECK(children.contains(table_column_name));

be/src/io/fs/connectivity/s3_connectivity_tester.cpp

@@ -24,6 +24,10 @@ namespace doris::io {
 
 Status S3ConnectivityTester::test(const std::map<std::string, std::string>& properties) {
     auto it = properties.find(TEST_LOCATION);
+    if (it == properties.end()) {
+        return Status::InvalidArgument("Missing '{}' property in S3 connectivity test request",
+                                       TEST_LOCATION);
+    }
     S3URI s3_uri(it->second);
     RETURN_IF_ERROR(s3_uri.parse());
 

be/test/format/table/table_schema_change_helper_test.cpp

@@ -167,6 +167,43 @@ TEST(MockTableSchemaChangeHelper, OrcNameNoSchemaChange) {
               "    ScalarNode\n");
 }
 
+TEST(MockTableSchemaChangeHelper, HasChildrenColumnGuardsAbsentProjectedColumn) {
+    // Build a top-level StructNode with two known columns via the public by_orc_name path.
+    SlotDescriptor slot1;
+    slot1._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true);
+    slot1._col_name = "col1";
+
+    SlotDescriptor slot2;
+    slot2._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true);
+    slot2._col_name = "col2";
+
+    TupleDescriptor tuple_desc;
+    tuple_desc.add_slot(&slot1);
+    tuple_desc.add_slot(&slot2);
+
+    std::unique_ptr<orc::Type> orc_type(
+            orc::Type::buildTypeFromString("struct<col1:int,col2:int>"));
+    std::shared_ptr<TableSchemaChangeHelper::Node> node = nullptr;
+    ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_name(&tuple_desc,
+                                                                         orc_type.get(), node)
+                        .ok());
+
+    // A projected column that IS in the table schema tree -> present.
+    EXPECT_TRUE(node->has_children_column("col1"));
+    EXPECT_TRUE(node->has_children_column("col2"));
+    // A projected column that is NOT in the tree at all models an FE/BE schema-contract mismatch
+    // (e.g. a paimon renamed column missing from a stale history_schema_info entry). has_children_column
+    // must report it absent WITHOUT the DCHECK/abort that children_column_exists would trigger — this is
+    // the presence check the parquet reader's missing-cols guard relies on to fail the query instead of
+    // aborting the whole BE. MUTATION: having has_children_column call children.at() -> abort/red.
+    EXPECT_FALSE(node->has_children_column("not_a_projected_column"));
+
+    // ConstNode (the no-schema-change path) reports every column present, matching its
+    // children_column_exists.
+    EXPECT_TRUE(
+            TableSchemaChangeHelper::ConstNode::get_instance()->has_children_column("anything"));
+}
+
 TEST(MockTableSchemaChangeHelper, OrcNameSchemaChange1) {
     std::vector<DataTypePtr> data_types;
     std::vector<std::string> column_names;

build.sh

@@ -1041,7 +1041,11 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     mkdir -p "${DORIS_OUTPUT}/fe/conf/ssl"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/jdbc_drivers/"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_udf/"
-    mkdir -p "${DORIS_OUTPUT}/fe/plugins/connectors/"
+    # Drop point for the trino-connector's own Trino plugins. Deliberately NOT the legacy
+    # plugins/connectors/: that name is still read as a fallback for deployments upgrading from
+    # <= 2.1.8, so a fresh install must not create it (an empty dir would be harmless, but the
+    # one-letter gap to the plugins/connector/ tree above is not).
+    mkdir -p "${DORIS_OUTPUT}/fe/plugins/trino_plugins/"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/hadoop_conf/"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_extensions/"
 
@@ -1081,6 +1085,23 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     done
     unset CONN_PLUGIN_DIR conn_module conn_plugin_target conn_module_dir conn_zip
 
+    # RC-4: self-contain the paimon connector plugin for OSS. The connector sets
+    # fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem; that impl lives in the jindofs jars,
+    # which are packaged from thirdparty by post-build.sh into fe/lib/jindofs (NOT a maven artifact).
+    # The plugin runs child-first, so without its OWN copy JindoOssFileSystem resolves from the parent
+    # 'app' classloader and cannot be cast to the plugin's child-loaded org.apache.hadoop.fs.FileSystem.
+    # Copy the jindofs jars into the paimon plugin lib so JindoOssFileSystem loads child-first alongside
+    # the plugin's own hadoop FileSystem (same self-contained intent as the bundled hadoop-aws/S3A).
+    # Naturally gated: a no-op unless jindofs was packaged (--jindofs / DISABLE_BUILD_JINDOFS=OFF).
+    # CAVEAT (docker-gated, enablePaimonTest=true): jindo-core ships a native lib that can bind to only one
+    # classloader per JVM, so this is safe only while no concurrent non-paimon path loads jindo from
+    # fe/lib/jindofs in the same FE process.
+    PAIMON_CONN_LIB="${DORIS_OUTPUT}/fe/plugins/connector/paimon/lib"
+    if [[ -d "${PAIMON_CONN_LIB}" && -d "${DORIS_OUTPUT}/fe/lib/jindofs" ]]; then
+        cp -p "${DORIS_OUTPUT}/fe/lib/jindofs/"*.jar "${PAIMON_CONN_LIB}/" 2>/dev/null || true
+    fi
+    unset PAIMON_CONN_LIB
+
     if [ "${TARGET_SYSTEM}" = "Darwin" ] || [ "${TARGET_SYSTEM}" = "Linux" ]; then
       mkdir -p "${DORIS_OUTPUT}/fe/arthas"
       rm -rf "${DORIS_OUTPUT}/fe/arthas/*"
@@ -1249,7 +1270,8 @@ EOF
     mkdir -p "${DORIS_OUTPUT}/be/plugins/jdbc_drivers/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/java_udf/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/python_udf/"
-    mkdir -p "${DORIS_OUTPUT}/be/plugins/connectors/"
+    # Mirrors the FE drop point above; the BE JNI scanner loads the same Trino plugins independently.
+    mkdir -p "${DORIS_OUTPUT}/be/plugins/trino_plugins/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/hadoop_conf/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/java_extensions/"
     cp -r -p "${DORIS_HOME}/be/src/udf/python/python_server.py" "${DORIS_OUTPUT}/be/plugins/python_udf/"

docker/thirdparties/custom_settings.env

@@ -20,7 +20,13 @@
 # Do not use "_" or other sepcial characters, only number and alphabeta.
 # eg: CONTAINER_UID="doris-jack-"
 # NOTICE: change this uid will modify the file in docker-compose.
+# NOTICE: the CI pipeline rewrites the assignment below by exact text match
+#         (sed 's/^CONTAINER_UID="doris--"/CONTAINER_UID="doris-external--"/'),
+#         so that line must stay literal. Put overrides on the lines around it.
+CONTAINER_UID_FROM_ENV="${CONTAINER_UID:-}"
 CONTAINER_UID="doris--"
+CONTAINER_UID="${CONTAINER_UID_FROM_ENV:-${CONTAINER_UID}}"
+unset CONTAINER_UID_FROM_ENV
 export s3Endpoint="oss-cn-hongkong.aliyuncs.com"
 export s3BucketName="doris-regression-hk"
 

docker/thirdparties/docker-compose/iceberg-rest/docker-compose.yaml.tpl

@@ -156,6 +156,7 @@ services:
   # Built-in HDFS NameNode
   hdfs-namenode:
     image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
+    platform: linux/amd64
     container_name: ${CONTAINER_UID}iceberg-hdfs-namenode
     restart: always
     ports:
@@ -186,6 +187,7 @@ services:
   # Built-in HDFS DataNode
   hdfs-datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
+    platform: linux/amd64
     container_name: ${CONTAINER_UID}iceberg-hdfs-datanode
     restart: always
     ports:

docker/thirdparties/docker-compose/iceberg-rest/iceberg-rest_settings.env

@@ -39,8 +39,23 @@ if command -v ip >/dev/null 2>&1; then
     # Linux: get default route interface IP
     export LOCAL_IP=$(ip route get 1.1.1.1 | grep -oP 'src \K\S+' 2>/dev/null || echo "127.0.0.1")
 elif command -v ifconfig >/dev/null 2>&1; then
-    # macOS: get en0 interface IP
-    export LOCAL_IP=$(ifconfig en0 | grep 'inet ' | awk '{print $2}' 2>/dev/null || echo "127.0.0.1")
+    # macOS: resolve the active default-route interface instead of assuming en0
+    DEFAULT_INTERFACE=$(route -n get default 2>/dev/null | awk '/interface:/{print $2}')
+    export LOCAL_IP=$(ipconfig getifaddr "${DEFAULT_INTERFACE}" 2>/dev/null || true)
+    if [[ -z "${LOCAL_IP}" ]]; then
+        export LOCAL_IP=$(ifconfig | awk '
+            /^[a-z0-9]+:/ {
+                interface_name = $1
+                sub(/:$/, "", interface_name)
+            }
+            /^[[:space:]]+inet / &&
+                    interface_name !~ /^(lo|utun|awdl|llw|bridge)/ {
+                print $2
+                exit
+            }
+        ')
+    fi
+    export LOCAL_IP="${LOCAL_IP:-127.0.0.1}"
 else
     # Fallback to localhost
     export LOCAL_IP="127.0.0.1"