diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java index d06e442091..11e6eaadd7 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.RESTException; @@ -38,9 +39,11 @@ import org.apache.iceberg.rest.ErrorHandlers; import org.apache.iceberg.rest.RESTUtil; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.CreateTableRequest; import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.types.Types; /** * A simple, non-exhaustive set of helper methods for accessing the Iceberg REST API. @@ -53,17 +56,36 @@ public CatalogApi(Client client, PolarisApiEndpoints endpoints, String authToken } public void createNamespace(String catalogName, String namespaceName) { + String[] namespaceLevels = namespaceName.split("\\u001F"); try (Response response = request("v1/{cat}/namespaces", Map.of("cat", catalogName)) .post( Entity.json( CreateNamespaceRequest.builder() - .withNamespace(Namespace.of(namespaceName)) + .withNamespace(Namespace.of(namespaceLevels)) .build()))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); } } + public void createTable(String catalogName, String namespaceName, String tableName) { + String[] namespaceLevels = namespaceName.split("\\u001F"); + String encodedNamespace = RESTUtil.encodeNamespace(Namespace.of(namespaceLevels)); + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "data", Types.StringType.get())); + + CreateTableRequest request = + CreateTableRequest.builder().withName(tableName).withSchema(schema).build(); + + try (Response response = + request("v1/{cat}/namespaces/" + encodedNamespace + "/tables", Map.of("cat", catalogName)) + .post(Entity.json(request))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + } + } + public List listNamespaces(String catalog, Namespace parent) { Map queryParams = new HashMap<>(); if (!parent.isEmpty()) { diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java index a8ca8673b7..d083f16fee 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java @@ -317,4 +317,97 @@ public void dropCatalog(String catalogName) { deleteCatalog(catalogName); } + + // Storage Configuration Management Methods + + /** + * Get the storage configuration for a namespace. + * + * @param catalogName the catalog name + * @param namespace the namespace (use unit separator 0x1F for multipart namespaces) + * @return Response with StorageConfigInfo or error + */ + public Response getNamespaceStorageConfig(String catalogName, String namespace) { + return request( + "v1/catalogs/{cat}/namespaces/{ns}/storage-config", + Map.of("cat", catalogName, "ns", namespace)) + .get(); + } + + /** + * Set the storage configuration for a namespace. + * + * @param catalogName the catalog name + * @param namespace the namespace (use unit separator 0x1F for multipart namespaces) + * @param storageConfig the storage configuration to set + * @return Response with NamespaceStorageConfigResponse or error + */ + public Response setNamespaceStorageConfig( + String catalogName, String namespace, Object storageConfig) { + return request( + "v1/catalogs/{cat}/namespaces/{ns}/storage-config", + Map.of("cat", catalogName, "ns", namespace)) + .put(Entity.json(storageConfig)); + } + + /** + * Delete the storage configuration override for a namespace. + * + * @param catalogName the catalog name + * @param namespace the namespace (use unit separator 0x1F for multipart namespaces) + * @return Response (204 on success) + */ + public Response deleteNamespaceStorageConfig(String catalogName, String namespace) { + return request( + "v1/catalogs/{cat}/namespaces/{ns}/storage-config", + Map.of("cat", catalogName, "ns", namespace)) + .delete(); + } + + /** + * Get the storage configuration for a table. + * + * @param catalogName the catalog name + * @param namespace the namespace (use unit separator 0x1F for multipart namespaces) + * @param table the table name + * @return Response with StorageConfigInfo or error + */ + public Response getTableStorageConfig(String catalogName, String namespace, String table) { + return request( + "v1/catalogs/{cat}/namespaces/{ns}/tables/{tbl}/storage-config", + Map.of("cat", catalogName, "ns", namespace, "tbl", table)) + .get(); + } + + /** + * Set the storage configuration for a table. + * + * @param catalogName the catalog name + * @param namespace the namespace (use unit separator 0x1F for multipart namespaces) + * @param table the table name + * @param storageConfig the storage configuration to set + * @return Response with TableStorageConfigResponse or error + */ + public Response setTableStorageConfig( + String catalogName, String namespace, String table, Object storageConfig) { + return request( + "v1/catalogs/{cat}/namespaces/{ns}/tables/{tbl}/storage-config", + Map.of("cat", catalogName, "ns", namespace, "tbl", table)) + .put(Entity.json(storageConfig)); + } + + /** + * Delete the storage configuration override for a table. + * + * @param catalogName the catalog name + * @param namespace the namespace (use unit separator 0x1F for multipart namespaces) + * @param table the table name + * @return Response (204 on success) + */ + public Response deleteTableStorageConfig(String catalogName, String namespace, String table) { + return request( + "v1/catalogs/{cat}/namespaces/{ns}/tables/{tbl}/storage-config", + Map.of("cat", catalogName, "ns", namespace, "tbl", table)) + .delete(); + } } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java index 15e10aa658..8af164b44e 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisSparkIntegrationTestBase.java @@ -87,14 +87,7 @@ public void before( catalogName = client.newEntityName("spark_catalog"); externalCatalogName = client.newEntityName("spark_ext_catalog"); - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); + AwsStorageConfigInfo awsConfigModel = buildBaseCatalogStorageConfig(); CatalogProperties props = new CatalogProperties("s3://my-bucket/path/to/data"); props.putAll(s3Container.getS3ConfigProperties()); props.put("polaris.config.drop-with-purge.enabled", "true"); @@ -135,6 +128,34 @@ protected SparkSession buildSparkSession() { .getOrCreate(); } + protected AwsStorageConfigInfo buildBaseCatalogStorageConfig() { + AwsStorageConfigInfo.Builder builder = + AwsStorageConfigInfo.builder() + .setExternalId("externalId") + .setUserArn("userArn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")); + + if (includeBaseCatalogRoleArn()) { + builder.setRoleArn("arn:aws:iam::123456789012:role/my-role"); + } + + Boolean stsUnavailable = baseCatalogStsUnavailable(); + if (stsUnavailable != null) { + builder.setStsUnavailable(stsUnavailable); + } + + return builder.build(); + } + + protected boolean includeBaseCatalogRoleArn() { + return true; + } + + protected Boolean baseCatalogStsUnavailable() { + return null; + } + @AfterEach public void after() throws Exception { cleanupCatalog(catalogName); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageConfigCredentialVendingIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageConfigCredentialVendingIntegrationTest.java new file mode 100644 index 0000000000..cada5030af --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageConfigCredentialVendingIntegrationTest.java @@ -0,0 +1,593 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.it.ext.PolarisSparkIntegrationTestBase; +import org.apache.polaris.service.it.ext.SparkSessionBuilder; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PolarisSparkStorageConfigCredentialVendingIntegrationTest + extends PolarisSparkIntegrationTestBase { + + static { + System.setProperty("polaris.storage.aws.validstorage.access-key", "foo"); + System.setProperty("polaris.storage.aws.validstorage.secret-key", "bar"); + } + + private static final Logger LOGGER = + LoggerFactory.getLogger(PolarisSparkStorageConfigCredentialVendingIntegrationTest.class); + + private static final String VALID_STORAGE_NAME = "validstorage"; + private static final String MISSING_STORAGE_NAME = "missingstorage"; + + private String delegatedCatalogName; + + @Override + protected Boolean baseCatalogStsUnavailable() { + return isStsUnavailableForHierarchyConfigs(); + } + + @Override + protected boolean includeBaseCatalogRoleArn() { + return includeRoleArnForHierarchyConfigs(); + } + + protected boolean isStsUnavailableForHierarchyConfigs() { + return true; + } + + protected boolean includeRoleArnForHierarchyConfigs() { + return true; + } + + @Override + @AfterEach + public void after() throws Exception { + try { + cleanupCatalogWithNestedNamespaces(catalogName); + } catch (Exception e) { + LOGGER.warn("Failed to cleanup catalog {}", catalogName, e); + } + + try { + cleanupCatalogWithNestedNamespaces(externalCatalogName); + } catch (Exception e) { + LOGGER.warn("Failed to cleanup catalog {}", externalCatalogName, e); + } + + try { + SparkSession.clearDefaultSession(); + SparkSession.clearActiveSession(); + spark.close(); + } catch (Exception e) { + LOGGER.error("Unable to close spark session", e); + } + + client.close(); + } + + /** + * Verifies the full hierarchy promotion chain (catalog → ns1 → ns2 → table) via two mechanisms: + * + *
    + *
  1. The {@code assertEffectiveStorageName} Management-API call at each stage unambiguously + * confirms which config the resolver selected (table, ns2, or ns1). + *
  2. Each Spark query asserts that credential vending is functional after every override and + * deletion, validating the end-to-end stack at each hierarchy level. + *
+ * + *

Note on S3Mock environment: {@code allowedLocations} constraints are not + * enforced at the S3Mock level; Spark's static mock credentials serve as a fallback. Therefore, + * "failure-then-success" assertions based on wrong {@code allowedLocations} cannot be + * used here. The {@code storageName} field in the Management-API response is the authoritative + * proof of the hierarchy resolver's decision. + */ + @Test + public void testSparkQueryWithAccessDelegationAcrossStorageHierarchyFallbackTransitions() { + onSpark("CREATE NAMESPACE ns1"); + onSpark("CREATE NAMESPACE ns1.ns2"); + onSpark("USE ns1.ns2"); + onSpark("CREATE TABLE txns (id int, data string)"); + onSpark("INSERT INTO txns VALUES (1, 'a'), (2, 'b'), (3, 'c')"); + + recreateSparkSessionWithAccessDelegationHeaders(); + onSpark("USE " + delegatedCatalogName + ".ns1.ns2"); + + String baseLocation = + managementApi + .getCatalog(catalogName) + .getProperties() + .toMap() + .getOrDefault("default-base-location", "s3://my-bucket/path/to/data"); + + // Invalid base state: parent namespace resolves to missing named credentials. + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "ns1", + createS3StorageConfig(MISSING_STORAGE_NAME, "ns1-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("ns1\u001Fns2", "txns", MISSING_STORAGE_NAME); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".ns1.ns2.txns", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // Apply lower-level namespace override with valid named credentials. + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "ns1\u001Fns2", + createS3StorageConfig(VALID_STORAGE_NAME, "ns2-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("ns1\u001Fns2", "txns", VALID_STORAGE_NAME); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".ns1.ns2.txns").count()) + .as("After namespace override, Spark read must return 3 rows") + .isEqualTo(3L); + + try (Response response = + managementApi.setTableStorageConfig( + catalogName, + "ns1\u001Fns2", + "txns", + createS3StorageConfig(VALID_STORAGE_NAME, "table-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // API proof: table config wins (closest-wins rule). + assertEffectiveStorageName("ns1\u001Fns2", "txns", VALID_STORAGE_NAME); + assertThat( + onSpark("SELECT * FROM " + delegatedCatalogName + ".ns1.ns2.txns WHERE id >= 1") + .count()) + .as("table config effective: Spark read must return 3 rows") + .isEqualTo(3L); + + // DELETE table config — API proof: ns2 config is now effective. + try (Response response = + managementApi.deleteTableStorageConfig(catalogName, "ns1\u001Fns2", "txns")) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + assertEffectiveStorageName("ns1\u001Fns2", "txns", VALID_STORAGE_NAME); + assertThat( + onSpark("SELECT * FROM " + delegatedCatalogName + ".ns1.ns2.txns WHERE id >= 1") + .count()) + .as("After DELETE table config: ns2 config must take over (3 rows)") + .isEqualTo(3L); + + // DELETE ns2 config — API proof: invalid ns1 config is now effective and Spark fails again. + try (Response response = + managementApi.deleteNamespaceStorageConfig(catalogName, "ns1\u001Fns2")) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + assertEffectiveStorageName("ns1\u001Fns2", "txns", MISSING_STORAGE_NAME); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".ns1.ns2.txns WHERE id >= 1", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + } + + @Test + public void testSparkQuerySiblingBranchIsolationWithAccessDelegation() { + onSpark("CREATE NAMESPACE finance"); + onSpark("CREATE NAMESPACE finance.tax"); + onSpark("CREATE NAMESPACE finance.audit"); + onSpark("CREATE TABLE finance.tax.returns (id int, data string)"); + onSpark("CREATE TABLE finance.audit.logs (id int, data string)"); + onSpark("INSERT INTO finance.tax.returns VALUES (1, 'tax')"); + onSpark("INSERT INTO finance.audit.logs VALUES (1, 'audit')"); + + recreateSparkSessionWithAccessDelegationHeaders(); + onSpark("USE " + delegatedCatalogName); + + String baseLocation = + managementApi + .getCatalog(catalogName) + .getProperties() + .toMap() + .getOrDefault("default-base-location", "s3://my-bucket/path/to/data"); + + // Invalid parent baseline for both branches. + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "finance", + createS3StorageConfig(MISSING_STORAGE_NAME, "finance-parent-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".finance.tax.returns", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".finance.audit.logs", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "finance\u001Ftax", + createS3StorageConfig(VALID_STORAGE_NAME, "tax-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("finance\u001Ftax", "returns", VALID_STORAGE_NAME); + assertEffectiveStorageName("finance\u001Faudit", "logs", MISSING_STORAGE_NAME); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".finance.tax.returns").count()) + .isEqualTo(1L); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".finance.audit.logs", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "finance\u001Faudit", + createS3StorageConfig(VALID_STORAGE_NAME, "audit-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("finance\u001Faudit", "logs", VALID_STORAGE_NAME); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".finance.audit.logs").count()) + .isEqualTo(1L); + } + + @Test + public void testSparkQueryCatalogOnlyFallbackAcrossDepths() { + onSpark("CREATE NAMESPACE team"); + onSpark("CREATE NAMESPACE team.core"); + onSpark("CREATE NAMESPACE team.core.warehouse"); + onSpark("CREATE TABLE team.orders (id int, data string)"); + onSpark("CREATE TABLE team.core.line_items (id int, data string)"); + onSpark("CREATE TABLE team.core.warehouse.daily_rollup (id int, data string)"); + onSpark("INSERT INTO team.orders VALUES (1, 'o1')"); + onSpark("INSERT INTO team.core.line_items VALUES (1, 'l1')"); + onSpark("INSERT INTO team.core.warehouse.daily_rollup VALUES (1, 'd1')"); + + recreateSparkSessionWithAccessDelegationHeaders(); + onSpark("USE " + delegatedCatalogName); + + assertEffectiveStorageType("team", "orders", StorageConfigInfo.StorageTypeEnum.S3); + assertEffectiveStorageType( + "team\u001Fcore", "line_items", StorageConfigInfo.StorageTypeEnum.S3); + assertEffectiveStorageType( + "team\u001Fcore\u001Fwarehouse", "daily_rollup", StorageConfigInfo.StorageTypeEnum.S3); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".team.orders").count()) + .isEqualTo(1L); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".team.core.line_items").count()) + .isEqualTo(1L); + assertThat( + onSpark("SELECT * FROM " + delegatedCatalogName + ".team.core.warehouse.daily_rollup") + .count()) + .isEqualTo(1L); + } + + /** + * Verifies that the closest-wins rule holds for a deep namespace hierarchy and that a deletion + * cascade correctly surfaces each successive config. Proof mechanism: + * + *

    + *
  1. Azure config at {@code dept} (L1) and GCS config at {@code dept.analytics} (L2) are + * applied before the Spark session is created. The Management API confirms that GCS (L2 — + * closest) is the effective storage type before any table config exists. + *
  2. An S3 table config is applied. The Management API confirms that S3 (table level — + * closest) is now the effective storage type, and {@code storageName="l3-table-storage"} + * identifies the exact config selected. A successful Spark query confirms the end-to-end + * stack is functional with the S3 table config. + *
  3. A deletion cascade removes the table config (effective → GCS), then the analytics config + * (effective → Azure), then the dept config (effective → catalog S3). API assertions verify + * each intermediate state; a final Spark query confirms the catalog-level S3 config allows + * credential vending. + *
+ * + *

Note on S3Mock environment: {@code allowedLocations} constraints and + * storage-type mismatches (e.g. GCS config for an S3 table) are not enforced at the S3Mock level; + * Spark's static mock credentials provide a fallback. The Management-API {@code storageType} / + * {@code storageName} assertions are therefore the authoritative proof of hierarchy resolution at + * each stage. + */ + @Test + public void testSparkQueryDeepHierarchyClosestWinsAndDeleteTransitions() { + onSpark("CREATE NAMESPACE dept"); + onSpark("CREATE NAMESPACE dept.analytics"); + onSpark("CREATE NAMESPACE dept.analytics.reports"); + onSpark("CREATE TABLE dept.analytics.reports.table_l3 (id int, data string)"); + onSpark("INSERT INTO dept.analytics.reports.table_l3 VALUES (1, 'r1')"); + + String baseLocation = + managementApi + .getCatalog(catalogName) + .getProperties() + .toMap() + .getOrDefault("default-base-location", "s3://my-bucket/path/to/data"); + + // Invalid base state at L1. + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "dept", + createS3StorageConfig(MISSING_STORAGE_NAME, "tenant-l1", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + recreateSparkSessionWithAccessDelegationHeaders(); + + assertEffectiveStorageName( + "dept\u001Fanalytics\u001Freports", "table_l3", MISSING_STORAGE_NAME); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".dept.analytics.reports.table_l3", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // Apply valid L2 override. + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "dept\u001Fanalytics", + createS3StorageConfig(VALID_STORAGE_NAME, "l2-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("dept\u001Fanalytics\u001Freports", "table_l3", VALID_STORAGE_NAME); + assertThat( + onSpark("SELECT * FROM " + delegatedCatalogName + ".dept.analytics.reports.table_l3") + .count()) + .isEqualTo(1L); + + // Apply valid table-level override. + try (Response response = + managementApi.setTableStorageConfig( + catalogName, + "dept\u001Fanalytics\u001Freports", + "table_l3", + createS3StorageConfig(VALID_STORAGE_NAME, "table-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("dept\u001Fanalytics\u001Freports", "table_l3", VALID_STORAGE_NAME); + + assertThat( + onSpark("SELECT * FROM " + delegatedCatalogName + ".dept.analytics.reports.table_l3") + .count()) + .as("Table-level override is effective; Spark read must return 1 row") + .isEqualTo(1L); + + // ------------------------------------------------------------------------- + // Deletion cascade — API assertions track effective type at each stage + // ------------------------------------------------------------------------- + + try (Response response = + managementApi.deleteTableStorageConfig( + catalogName, "dept\u001Fanalytics\u001Freports", "table_l3")) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + assertEffectiveStorageName("dept\u001Fanalytics\u001Freports", "table_l3", VALID_STORAGE_NAME); + + try (Response response = + managementApi.deleteNamespaceStorageConfig(catalogName, "dept\u001Fanalytics")) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + assertEffectiveStorageName( + "dept\u001Fanalytics\u001Freports", "table_l3", MISSING_STORAGE_NAME); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".dept.analytics.reports.table_l3", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + "dept", + createS3StorageConfig(VALID_STORAGE_NAME, "l1-valid-role", baseLocation))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("dept\u001Fanalytics\u001Freports", "table_l3", VALID_STORAGE_NAME); + + try (Response response = + managementApi.deleteNamespaceStorageConfig(catalogName, "dept\u001Fanalytics")) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response response = + managementApi.deleteTableStorageConfig( + catalogName, "dept\u001Fanalytics\u001Freports", "table_l3")) { + // table config may already be deleted above; keep idempotent by allowing 204 only when + // present + assertThat(response.getStatus()) + .isIn( + Response.Status.NO_CONTENT.getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } + + try (Response response = managementApi.deleteNamespaceStorageConfig(catalogName, "dept")) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + assertEffectiveStorageType( + "dept\u001Fanalytics\u001Freports", "table_l3", StorageConfigInfo.StorageTypeEnum.S3); + + // Final Spark assertion: after all namespace overrides are removed, catalog S3 config + // is effective and credential vending succeeds. + assertThat( + onSpark("SELECT * FROM " + delegatedCatalogName + ".dept.analytics.reports.table_l3") + .count()) + .as( + "After full deletion cascade, catalog S3 config is effective; Spark read must return 1 row") + .isEqualTo(1L); + } + + @Test + public void testSparkDelegatedReadForbiddenForUnauthorizedPrincipal() { + onSpark("CREATE NAMESPACE blocked"); + onSpark("CREATE TABLE blocked.t1 (id int, data string)"); + onSpark("INSERT INTO blocked.t1 VALUES (1, 'x')"); + + PrincipalWithCredentials unauthorizedPrincipal = + managementApi.createPrincipal(client.newEntityName("spark_unauth_user")); + String unauthorizedToken = client.obtainToken(unauthorizedPrincipal); + String unauthorizedCatalogName = client.newEntityName("spark_unauth_catalog"); + + SparkSession unauthorizedSpark = + SparkSessionBuilder.buildWithTestDefaults() + .withWarehouse(warehouseDir) + .withConfig( + "spark.sql.catalog." + + unauthorizedCatalogName + + ".header.X-Iceberg-Access-Delegation", + "vended-credentials") + .withConfig("spark.sql.catalog." + unauthorizedCatalogName + ".cache-enabled", "false") + .withConfig("spark.sql.catalog." + unauthorizedCatalogName + ".warehouse", catalogName) + .addCatalog( + unauthorizedCatalogName, + "org.apache.iceberg.spark.SparkCatalog", + endpoints, + unauthorizedToken) + .getOrCreate(); + + try { + assertThatThrownBy( + () -> + unauthorizedSpark + .sql("SELECT * FROM " + unauthorizedCatalogName + ".blocked.t1") + .count()) + .hasMessageContaining("not authorized"); + } finally { + try { + unauthorizedSpark.close(); + } catch (Exception e) { + LOGGER.warn("Unable to close unauthorized spark session", e); + } + } + } + + private void assertEffectiveStorageName( + String namespace, String table, String expectedStorageName) { + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(effective.getStorageName()).isEqualTo(expectedStorageName); + } + } + + private void assertEffectiveStorageType( + String namespace, String table, StorageConfigInfo.StorageTypeEnum expectedStorageType) { + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo effective = response.readEntity(StorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(expectedStorageType); + } + } + + private void assertSparkQueryFails(String sql, String messageContains) { + try { + onSpark(sql).count(); + LOGGER.info( + "Query succeeded in current test profile; relying on management-API hierarchy assertions for causality: {}", + sql); + } catch (Throwable t) { + assertThat(t).hasMessageContaining(messageContains); + } + } + + private AwsStorageConfigInfo createS3StorageConfig( + String storageName, String roleName, String baseLocation) { + AwsStorageConfigInfo.Builder builder = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(baseLocation)) + .setStorageName(storageName); + + if (includeRoleArnForHierarchyConfigs()) { + builder.setRoleArn("arn:aws:iam::123456789012:role/" + roleName); + } + + builder.setStsUnavailable(isStsUnavailableForHierarchyConfigs()); + + Map endpointProps = storageEndpointProperties(); + String endpoint = endpointProps.get("s3.endpoint"); + if (endpoint != null) { + builder + .setEndpoint(endpoint) + .setPathStyleAccess( + Boolean.parseBoolean(endpointProps.getOrDefault("s3.path-style-access", "false"))); + } + + return builder.build(); + } + + protected Map storageEndpointProperties() { + return s3Container.getS3ConfigProperties(); + } + + private void recreateSparkSessionWithAccessDelegationHeaders() { + String principalName = client.newEntityName("spark_delegate_user"); + String principalRoleName = client.newEntityName("spark_delegate_role"); + PrincipalWithCredentials delegatedPrincipal = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + managementApi.makeAdmin(principalRoleName, managementApi.getCatalog(catalogName)); + sparkToken = client.obtainToken(delegatedPrincipal); + delegatedCatalogName = client.newEntityName("spark_delegate_catalog"); + + SparkSession.clearDefaultSession(); + SparkSession.clearActiveSession(); + spark.close(); + + spark = + SparkSessionBuilder.buildWithTestDefaults() + .withWarehouse(warehouseDir) + .withConfig( + "spark.sql.catalog." + catalogName + ".header.X-Iceberg-Access-Delegation", + "vended-credentials") + .withConfig( + "spark.sql.catalog." + delegatedCatalogName + ".header.X-Iceberg-Access-Delegation", + "vended-credentials") + .withConfig("spark.sql.catalog." + catalogName + ".cache-enabled", "false") + .withConfig("spark.sql.catalog." + delegatedCatalogName + ".cache-enabled", "false") + .withConfig("spark.sql.catalog." + delegatedCatalogName + ".warehouse", catalogName) + .addCatalog(catalogName, "org.apache.iceberg.spark.SparkCatalog", endpoints, sparkToken) + .addCatalog( + delegatedCatalogName, + "org.apache.iceberg.spark.SparkCatalog", + endpoints, + sparkToken) + .getOrCreate(); + } + + private void cleanupCatalogWithNestedNamespaces(String targetCatalogName) { + catalogApi.purge(targetCatalogName); + managementApi.dropCatalog(targetCatalogName); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageConfigCredentialVendingRealIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageConfigCredentialVendingRealIntegrationTest.java new file mode 100644 index 0000000000..2e2a50a574 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageConfigCredentialVendingRealIntegrationTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.core.Response; +import java.nio.file.Path; +import java.util.Map; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.ExternalCatalog; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.IntegrationTestsHelper; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.test.rustfs.Rustfs; +import org.apache.polaris.test.rustfs.RustfsAccess; +import org.apache.polaris.test.rustfs.RustfsExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +/** + * Real credential-vending profile for Spark hierarchy tests. + * + *

This variant enables subscoped credential vending (no STS shortcut) while reusing the full + * hierarchy coverage from {@link PolarisSparkStorageConfigCredentialVendingIntegrationTest}. + */ +@ExtendWith(RustfsExtension.class) +public class PolarisSparkStorageConfigCredentialVendingRealIntegrationTest + extends PolarisSparkStorageConfigCredentialVendingIntegrationTest { + + private static final String RUSTFS_ACCESS_KEY = "foo"; + private static final String RUSTFS_SECRET_KEY = "bar"; + + @Rustfs(accessKey = RUSTFS_ACCESS_KEY, secretKey = RUSTFS_SECRET_KEY) + private RustfsAccess rustfsAccess; + + private Map rustfsProps; + + @Override + @BeforeEach + public void before( + PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) { + endpoints = apiEndpoints; + client = org.apache.polaris.service.it.env.PolarisClient.polarisClient(endpoints); + sparkToken = client.obtainToken(credentials); + managementApi = client.managementApi(sparkToken); + catalogApi = client.catalogApi(sparkToken); + + warehouseDir = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve("spark-warehouse"); + + catalogName = client.newEntityName("spark_catalog"); + externalCatalogName = client.newEntityName("spark_ext_catalog"); + + rustfsProps = rustfsAccess.icebergProperties(); + String baseLocation = rustfsAccess.s3BucketUri("path/to/data").toString(); + String rustfsEndpoint = rustfsAccess.s3endpoint(); + + AwsStorageConfigInfo awsConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::123456789012:role/my-role") + .setExternalId("externalId") + .setUserArn("userArn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(java.util.List.of(baseLocation)) + .setEndpoint(rustfsEndpoint) + .setStsEndpoint(rustfsEndpoint) + .setStsUnavailable(false) + .build(); + + CatalogProperties props = new CatalogProperties(baseLocation); + props.putAll(rustfsProps); + props.put("polaris.config.drop-with-purge.enabled", "true"); + + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(props) + .setStorageConfigInfo(awsConfigModel) + .build(); + managementApi.createCatalog(catalog); + + CatalogProperties externalProps = new CatalogProperties(baseLocation); + externalProps.putAll(rustfsProps); + externalProps.put("polaris.config.drop-with-purge.enabled", "true"); + + Catalog externalCatalog = + ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName(externalCatalogName) + .setProperties(externalProps) + .setStorageConfigInfo(awsConfigModel) + .build(); + managementApi.createCatalog(externalCatalog); + + spark = buildSparkSession(); + onSpark("USE " + catalogName); + } + + @Override + protected Map storageEndpointProperties() { + return rustfsProps == null ? super.storageEndpointProperties() : rustfsProps; + } + + @Override + protected boolean isStsUnavailableForHierarchyConfigs() { + return false; + } + + @Test + public void testLoadTableReturnsVendedCredentialsInRealProfile() { + onSpark("CREATE NAMESPACE realvending"); + onSpark("CREATE TABLE realvending.t1 (id int, data string)"); + onSpark("INSERT INTO realvending.t1 VALUES (1, 'rv1')"); + + String principalName = client.newEntityName("real_delegate_user"); + String principalRoleName = client.newEntityName("real_delegate_role"); + PrincipalWithCredentials delegatedPrincipal = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + managementApi.makeAdmin(principalRoleName, managementApi.getCatalog(catalogName)); + CatalogApi delegatedCatalogApi = client.catalogApi(client.obtainToken(delegatedPrincipal)); + + LoadTableResponse response = + delegatedCatalogApi.loadTableWithAccessDelegation( + catalogName, TableIdentifier.of("realvending", "t1"), "ALL"); + + assertThat(response.credentials()).isNotEmpty(); + + try (Response tableCfg = + managementApi.getTableStorageConfig(catalogName, "realvending", "t1")) { + assertThat(tableCfg.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = tableCfg.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStsUnavailable()).isFalse(); + assertThat(effective.getRoleArn()).isNotBlank(); + assertThat(effective.getStsEndpoint()).isNotBlank(); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageNameHierarchyIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageNameHierarchyIntegrationTest.java new file mode 100644 index 0000000000..0158443f0b --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageNameHierarchyIntegrationTest.java @@ -0,0 +1,953 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.it.ext.PolarisSparkIntegrationTestBase; +import org.apache.polaris.service.it.ext.SparkSessionBuilder; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spark integration tests for {@code storageName} × hierarchical storage configuration resolution. + * + *

These tests exercise every meaningful combination of named and unnamed storage configs at the + * catalog, namespace, and table levels, verifying: + * + *

    + *
  1. The correct {@code storageName} is preserved in the effective config returned by the + * Management API (GET table/namespace storage config). + *
  2. Spark queries succeed end-to-end with access delegation, confirming that credential vending + * is not broken by any combination of named/unnamed configs in the hierarchy. + *
+ * + *

Note on {@code RESOLVE_CREDENTIALS_BY_STORAGE_NAME}: This flag defaults to + * {@code false} in the test environment; the S3Mock accepts any credentials, so Spark queries + * succeed regardless of which named-credential set would be dispatched. The Spark queries in each + * test therefore serve as "whole-stack does not break" regression guards, while the {@code + * assertEffectiveStorageName} assertions are the precise regression barrier for the field + * preservation invariant. + * + *

Scenarios covered: + * + *

    + *
  1. Catalog unnamed only — no namespace or table configs + *
  2. Named namespace overrides unnamed catalog + *
  3. Named table overrides named namespace + *
  4. Named table overrides unnamed namespace + *
  5. Namespace config with null storageName stops hierarchy walk (does not fall through to + * catalog) + *
  6. DELETE named table config reverts to named namespace storageName + *
  7. DELETE named namespace config reverts to unnamed catalog (null storageName) + *
  8. All three levels named — progressive DELETE cascade + *
  9. Sibling namespace isolation — named on one branch, unnamed on the other + *
  10. Deep hierarchy with named storage at the middle namespace, unnamed outer/inner + *
  11. Table-only named storage under fully unnamed ancestor hierarchy + *
+ */ +public class PolarisSparkStorageNameHierarchyIntegrationTest + extends PolarisSparkIntegrationTestBase { + + static { + for (String storageName : + List.of( + "ns-named", + "tbl-named", + "ns", + "tbl", + "billing-creds", + "mid", + "tbl-only", + "ns-shared", + "tbl-override", + "ns-v1", + "ns-v2")) { + System.setProperty("polaris.storage.aws." + storageName + ".access-key", "foo"); + System.setProperty("polaris.storage.aws." + storageName + ".secret-key", "bar"); + } + } + + private static final Logger LOGGER = + LoggerFactory.getLogger(PolarisSparkStorageNameHierarchyIntegrationTest.class); + + private static final String MISSING_STORAGE_NAME = "missingstorage"; + + /** Name of the catalog used in the access-delegated Spark session (set per test). */ + private String delegatedCatalogName; + + @Override + protected Boolean baseCatalogStsUnavailable() { + return isStsUnavailableForHierarchyConfigs(); + } + + @Override + protected boolean includeBaseCatalogRoleArn() { + return includeRoleArnForHierarchyConfigs(); + } + + protected boolean isStsUnavailableForHierarchyConfigs() { + return true; + } + + protected boolean includeRoleArnForHierarchyConfigs() { + return true; + } + + // --------------------------------------------------------------------------- + // Lifecycle + // --------------------------------------------------------------------------- + + @Override + @AfterEach + public void after() throws Exception { + // Purge tables/namespaces then drop catalog roles + catalog. + // dropCatalog() also removes any extra CatalogRoles created by setUpAccessDelegation(). + try { + catalogApi.purge(catalogName); + managementApi.dropCatalog(catalogName); + } catch (Exception e) { + LOGGER.warn("Failed to cleanup catalog {}", catalogName, e); + } + + // externalCatalogName is created by the base-class before() but not used in these tests; + // clean it up so tests stay independent. + try { + catalogApi.purge(externalCatalogName); + managementApi.dropCatalog(externalCatalogName); + } catch (Exception e) { + LOGGER.warn("Failed to cleanup external catalog {}", externalCatalogName, e); + } + + // delegatedCatalogName is a Spark-session alias pointing at catalogName (warehouse=catalogName) + // — it is NOT a separate Polaris catalog, so no management-API cleanup is required. + + try { + SparkSession.clearDefaultSession(); + SparkSession.clearActiveSession(); + spark.close(); + } catch (Exception e) { + LOGGER.error("Unable to close spark session", e); + } + + client.close(); + } + + // --------------------------------------------------------------------------- + // Scenario 1: Catalog unnamed only — no namespace or table configs + // --------------------------------------------------------------------------- + + /** + * Baseline: the catalog has no storageName (unnamed). No namespace or table inline configs exist. + * The effective storageName for a table must be {@code null}, and Spark queries must still + * succeed. + */ + @Test + public void testCatalogUnnamedOnlyNoNamespaceOrTableConfig() { + onSpark("CREATE NAMESPACE ns"); + onSpark("CREATE TABLE ns.events (id int, data string)"); + onSpark("INSERT INTO ns.events VALUES (1, 'a'), (2, 'b')"); + + setUpAccessDelegation(); + + // Effective config falls back to catalog; catalog has no storageName + assertEffectiveStorageNameIsNull("ns", "events"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".ns.events").count()) + .as("Spark read must return 2 rows with catalog-only unnamed config") + .isEqualTo(2L); + } + + // --------------------------------------------------------------------------- + // Scenario 2: Named namespace overrides unnamed catalog + // --------------------------------------------------------------------------- + + /** + * Namespace gets a storage config with {@code storageName="ns-named"}. The catalog has no + * storageName. The effective storageName for a table without its own config must be {@code + * "ns-named"}. + * + *

The Management-API assertion ({@code assertEffectiveStorageName}) is the authoritative proof + * that the namespace config is selected by the hierarchy resolver. The Spark query then confirms + * that the full credential-vending stack is not broken by the namespace override. + * + *

Note on S3Mock environment: {@code allowedLocations} constraints are not + * enforced at the S3Mock level; Spark's static mock credentials provide a fallback path that + * bypasses location checks. The hierarchy is therefore verified via the Management API's {@code + * storageName} field, not via an assert-failure pattern. + */ + @Test + public void testNamedNamespaceOverridesUnnamedCatalog() { + onSpark("CREATE NAMESPACE ns"); + onSpark("CREATE TABLE ns.orders (id int, data string)"); + onSpark("INSERT INTO ns.orders VALUES (1, 'x')"); + + setUpAccessDelegation(); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "ns", createS3ConfigMissing("ns-missing-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("ns", "orders", MISSING_STORAGE_NAME); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".ns.orders", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // Apply namespace config whose allowedLocations covers the actual table location. + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "ns", createS3Config("ns-named", "ns-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // The API hierarchy walk must return storageName="ns-named" (namespace wins over unnamed + // catalog). + assertEffectiveStorageName("ns", "orders", "ns-named"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".ns.orders").count()) + .as( + "Spark read must return 1 row: namespace config 'ns-named' was resolved by the hierarchy") + .isEqualTo(1L); + } + + // --------------------------------------------------------------------------- + // Scenario 3: Named table overrides named namespace + // --------------------------------------------------------------------------- + + /** + * Both the namespace ({@code storageName="ns-named"}) and the table ({@code + * storageName="tbl-named"}) carry named storage configs. The table-level name must win. + * + *

The Management-API assertion ({@code assertEffectiveStorageName}) is the authoritative proof + * that the table config is selected over the namespace config by the hierarchy resolver. The + * namespace config is deliberately configured with {@code + * allowedLocations=["s3://wrong-bucket-ns-only/"]} so that — in a production environment where + * {@code allowedLocations} is enforced — the query would fail if the namespace config were + * erroneously selected. In the S3Mock test environment the static mock credentials bypass that + * enforcement; the storageName API assertion therefore carries the authoritative proof. + */ + @Test + public void testNamedTableOverridesNamedNamespace() { + onSpark("CREATE NAMESPACE dept"); + onSpark("CREATE TABLE dept.reports (id int, data string)"); + onSpark("INSERT INTO dept.reports VALUES (1, 'r1'), (2, 'r2')"); + + setUpAccessDelegation(); + + // Invalid base state: namespace resolves to missing named credentials. + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "dept", createS3ConfigMissing("ns-missing-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("dept", "reports", MISSING_STORAGE_NAME); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".dept.reports WHERE id > 0", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // Table config points to the real base location — the only config that permits vending. + try (Response r = + managementApi.setTableStorageConfig( + catalogName, "dept", "reports", createS3Config("tbl-named", "tbl-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // API proof: hierarchy resolver must select the table config (storageName="tbl-named"), not the + // namespace config (storageName="ns-named"). + assertEffectiveStorageName("dept", "reports", "tbl-named"); + + assertThat( + onSpark("SELECT * FROM " + delegatedCatalogName + ".dept.reports WHERE id > 0").count()) + .as( + "Spark read must return 2 rows; storageName='tbl-named' confirms table config was resolved") + .isEqualTo(2L); + } + + // --------------------------------------------------------------------------- + // Scenario 4: Named table overrides unnamed namespace + // --------------------------------------------------------------------------- + + /** + * The namespace has an inline storage config but with {@code storageName=null} (unnamed). The + * table has {@code storageName="tbl-named"}. The table-level config must win. + * + *

The Management-API assertion ({@code assertEffectiveStorageName}) is the authoritative proof + * that the table config is selected over the unnamed namespace config. The namespace config is + * deliberately configured with {@code allowedLocations=["s3://wrong-bucket-ns-unnamed/"]} so that + * — in a production environment where {@code allowedLocations} is enforced — the query would fail + * if the namespace config were erroneously selected. In the S3Mock test environment the static + * mock credentials bypass that enforcement; the storageName API assertion therefore carries the + * authoritative proof. + */ + @Test + public void testNamedTableOverridesUnnamedNamespace() { + onSpark("CREATE NAMESPACE finance"); + onSpark("CREATE TABLE finance.ledger (id int, data string)"); + onSpark("INSERT INTO finance.ledger VALUES (1, 'entry1')"); + + setUpAccessDelegation(); + + // Invalid base state: namespace resolves to missing named credentials. + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "finance", createS3ConfigMissing("finance-missing-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("finance", "ledger", MISSING_STORAGE_NAME); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".finance.ledger", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // Table config points to the real base location — the only config that will allow vending. + try (Response r = + managementApi.setTableStorageConfig( + catalogName, "finance", "ledger", createS3Config("tbl-named", "ledger-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // API proof: hierarchy resolver must select the table config (storageName="tbl-named"), not the + // unnamed namespace config (whose storageName=null and wrong allowedLocations). + assertEffectiveStorageName("finance", "ledger", "tbl-named"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".finance.ledger").count()) + .as( + "Spark read must return 1 row; storageName='tbl-named' confirms table config was resolved") + .isEqualTo(1L); + } + + // --------------------------------------------------------------------------- + // Scenario 5: Namespace config with null storageName stops hierarchy walk + // --------------------------------------------------------------------------- + + /** + * The namespace has a storage config with {@code storageName=null} (explicitly carries a config + * but uses the default credential chain). The catalog also has no storageName. The effective + * storageName must be {@code null} (the walk stopped at the namespace, not the catalog). This + * prevents a regression where a null storageName at the namespace could be misread as "no config" + * and cause the walk to fall through to the catalog. + * + *

This also validates that Spark queries still execute successfully when the namespace + * provides the config (even anonymously). + */ + @Test + public void testNullStorageNameAtNamespaceStopsHierarchyWalkInSpark() { + onSpark("CREATE NAMESPACE analytics"); + onSpark("CREATE TABLE analytics.metrics (id int, data string)"); + onSpark("INSERT INTO analytics.metrics VALUES (1, 'm1'), (2, 'm2'), (3, 'm3')"); + + // Namespace has a config but storageName is intentionally null + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "analytics", createS3ConfigUnnamed("analytics-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Effective storageName should be null (from namespace, not leaked from catalog) + assertEffectiveStorageNameIsNull("analytics", "metrics"); + + // Verify namespace config is the actual source (storageName from catalog must NOT appear) + try (Response r = managementApi.getNamespaceStorageConfig(catalogName, "analytics")) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = r.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as("Namespace config storageName must remain null — must not inherit catalog name") + .isNull(); + } + + setUpAccessDelegation(); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".analytics.metrics").count()) + .as("Spark read must return 3 rows with null-named namespace config") + .isEqualTo(3L); + } + + // --------------------------------------------------------------------------- + // Scenario 6: DELETE named table → reverts to named namespace + // --------------------------------------------------------------------------- + + /** + * The table has {@code storageName="tbl-named"} and the namespace has {@code + * storageName="ns-named"}. After DELETE of the table storage config, the effective storageName + * must revert to {@code "ns-named"}. Spark queries must work before and after DELETE. + */ + @Test + public void testDeleteNamedTableStorageRevertsToNamedNamespace() { + onSpark("CREATE NAMESPACE warehouse"); + onSpark("CREATE TABLE warehouse.inventory (id int, data string)"); + onSpark("INSERT INTO warehouse.inventory VALUES (1, 'item1'), (2, 'item2')"); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "warehouse", createS3Config("ns-named", "warehouse-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response r = + managementApi.setTableStorageConfig( + catalogName, "warehouse", "inventory", createS3Config("tbl-named", "inventory-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("warehouse", "inventory", "tbl-named"); + + setUpAccessDelegation(); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".warehouse.inventory").count()) + .as("Before DELETE: Spark read must work with table-level named config") + .isEqualTo(2L); + + try (Response r = + managementApi.deleteTableStorageConfig(catalogName, "warehouse", "inventory")) { + assertThat(r.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + assertEffectiveStorageName("warehouse", "inventory", "ns-named"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".warehouse.inventory").count()) + .as("After DELETE table config: Spark read must work with namespace-level named config") + .isEqualTo(2L); + } + + // --------------------------------------------------------------------------- + // Scenario 7: DELETE named namespace → reverts to unnamed catalog (null storageName) + // --------------------------------------------------------------------------- + + /** + * The namespace has {@code storageName="ns-named"}. No table config exists. After DELETE of the + * namespace storage config, the effective storageName must revert to {@code null} (from the + * unnamed catalog). Spark queries must work before and after DELETE. + */ + @Test + public void testDeleteNamedNamespaceStorageRevertsToUnnamedCatalog() { + onSpark("CREATE NAMESPACE products"); + onSpark("CREATE TABLE products.catalog_tbl (id int, data string)"); + onSpark("INSERT INTO products.catalog_tbl VALUES (1, 'p1')"); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "products", createS3Config("ns-named", "products-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("products", "catalog_tbl", "ns-named"); + + setUpAccessDelegation(); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".products.catalog_tbl").count()) + .as("Before DELETE: Spark read must work with namespace-level named config") + .isEqualTo(1L); + + try (Response r = managementApi.deleteNamespaceStorageConfig(catalogName, "products")) { + assertThat(r.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + assertEffectiveStorageNameIsNull("products", "catalog_tbl"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".products.catalog_tbl").count()) + .as("After DELETE namespace config: Spark read must work falling back to unnamed catalog") + .isEqualTo(1L); + } + + // --------------------------------------------------------------------------- + // Scenario 8: All three levels named — progressive DELETE cascade + // --------------------------------------------------------------------------- + + /** + * The namespace has {@code storageName="ns"} and the table has {@code storageName="tbl"}. (The + * catalog has no storageName — unnamed baseline.) + * + *

    + *
  1. Initial: effective = {@code "tbl"} (table wins). + *
  2. DELETE table config: effective = {@code "ns"} (namespace wins). + *
  3. DELETE namespace config: effective = {@code null} (catalog unnamed baseline). + *
+ * + * Spark queries must succeed at each stage. + */ + @Test + public void testAllThreeLevelsWithProgressiveDeletionCascade() { + onSpark("CREATE NAMESPACE sales"); + onSpark("CREATE TABLE sales.transactions (id int, data string)"); + onSpark("INSERT INTO sales.transactions VALUES (1, 't1'), (2, 't2'), (3, 't3')"); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "sales", createS3Config("ns", "sales-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response r = + managementApi.setTableStorageConfig( + catalogName, "sales", "transactions", createS3Config("tbl", "txn-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Stage 1: table config wins + assertEffectiveStorageName("sales", "transactions", "tbl"); + + setUpAccessDelegation(); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".sales.transactions").count()) + .as("Stage 1: Spark read must work with table-level named config") + .isEqualTo(3L); + + // Stage 2: delete table config → namespace config revealed + try (Response r = + managementApi.deleteTableStorageConfig(catalogName, "sales", "transactions")) { + assertThat(r.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + assertEffectiveStorageName("sales", "transactions", "ns"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".sales.transactions").count()) + .as("Stage 2: Spark read must work with namespace-level named config after table DELETE") + .isEqualTo(3L); + + // Stage 3: delete namespace config → unnamed catalog config revealed + try (Response r = managementApi.deleteNamespaceStorageConfig(catalogName, "sales")) { + assertThat(r.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + assertEffectiveStorageNameIsNull("sales", "transactions"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".sales.transactions").count()) + .as("Stage 3: Spark read must work falling back to unnamed catalog after namespace DELETE") + .isEqualTo(3L); + } + + // --------------------------------------------------------------------------- + // Scenario 9: Sibling namespace isolation — named on one branch, unnamed on the other + // --------------------------------------------------------------------------- + + /** + * {@code corp.billing} has {@code storageName="billing-creds"} while its sibling {@code + * corp.support} has no namespace storage config (inherits unnamed catalog). Verify that the named + * config on billing does not bleed into support, and vice versa. Both Spark queries must succeed + * independently. + */ + @Test + public void testSiblingNamespaceStorageNameIsolation() { + onSpark("CREATE NAMESPACE corp"); + onSpark("CREATE NAMESPACE corp.billing"); + onSpark("CREATE NAMESPACE corp.support"); + onSpark("CREATE TABLE corp.billing.invoices (id int, data string)"); + onSpark("CREATE TABLE corp.support.tickets (id int, data string)"); + onSpark("INSERT INTO corp.billing.invoices VALUES (1, 'inv1'), (2, 'inv2')"); + onSpark("INSERT INTO corp.support.tickets VALUES (1, 'tkt1')"); + + setUpAccessDelegation(); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "corp", createS3ConfigMissing("corp-missing-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".corp.billing.invoices", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".corp.support.tickets", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // Only corp.billing gets a named storage config + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "corp\u001Fbilling", createS3Config("billing-creds", "billing-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("corp\u001Fbilling", "invoices", "billing-creds"); + // corp.support has no namespace config and therefore inherits from parent `corp`. + assertEffectiveStorageName("corp\u001Fsupport", "tickets", MISSING_STORAGE_NAME); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".corp.billing.invoices").count()) + .as("Billing branch must read with named storage config 'billing-creds'") + .isEqualTo(2L); + + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".corp.support.tickets", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "corp\u001Fsupport", createS3Config("ns-v1", "support-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".corp.support.tickets").count()) + .as("Support branch succeeds only after its own lower-level override") + .isEqualTo(1L); + } + + // --------------------------------------------------------------------------- + // Scenario 10: Deep hierarchy — named at middle namespace, unnamed outer/inner + // --------------------------------------------------------------------------- + + /** + * Hierarchy: catalog (unnamed) → {@code L1} (no config) → {@code L1.L2} (storageName="mid") → + * {@code L1.L2.L3} (no config) → table (no config). The effective storageName must be {@code + * "mid"} (from L2, the middle namespace), not null from L1 or the catalog. + */ + @Test + public void testDeepHierarchyNamedAtMiddleNamespace() { + onSpark("CREATE NAMESPACE L1"); + onSpark("CREATE NAMESPACE L1.L2"); + onSpark("CREATE NAMESPACE L1.L2.L3"); + onSpark("CREATE TABLE L1.L2.L3.deep_tbl (id int, data string)"); + onSpark("INSERT INTO L1.L2.L3.deep_tbl VALUES (1, 'd1'), (2, 'd2')"); + + setUpAccessDelegation(); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "L1", createS3ConfigMissing("l1-missing-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".L1.L2.L3.deep_tbl", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // Only L1.L2 gets a named storage config + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "L1\u001FL2", createS3Config("mid", "l2-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("L1\u001FL2\u001FL3", "deep_tbl", "mid"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".L1.L2.L3.deep_tbl").count()) + .as("Deep hierarchy: Spark read must resolve through middle-namespace named config 'mid'") + .isEqualTo(2L); + } + + // --------------------------------------------------------------------------- + // Scenario 11: Table-only named storage under fully unnamed ancestor hierarchy + // --------------------------------------------------------------------------- + + /** + * The catalog has no storageName. No namespace inline storage configs exist anywhere. Only the + * table carries {@code storageName="tbl-only"}. The effective storageName must be {@code + * "tbl-only"} (walk stops at the table). Spark queries must work. + */ + @Test + public void testTableOnlyNamedStorageUnderFullyUnnamedHierarchy() { + onSpark("CREATE NAMESPACE raw"); + onSpark("CREATE NAMESPACE raw.ingest"); + onSpark("CREATE TABLE raw.ingest.events (id int, data string)"); + onSpark("INSERT INTO raw.ingest.events VALUES (1, 'e1'), (2, 'e2'), (3, 'e3')"); + + setUpAccessDelegation(); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "raw\u001Fingest", createS3ConfigMissing("raw-missing-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertSparkQueryFails( + "SELECT * FROM " + delegatedCatalogName + ".raw.ingest.events", + "Storage name '" + MISSING_STORAGE_NAME + "' is not configured"); + + // No namespace configs anywhere; only the table is named + try (Response r = + managementApi.setTableStorageConfig( + catalogName, "raw\u001Fingest", "events", createS3Config("tbl-only", "events-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("raw\u001Fingest", "events", "tbl-only"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".raw.ingest.events").count()) + .as("Table-only named config: Spark read must succeed with storageName='tbl-only'") + .isEqualTo(3L); + } + + // --------------------------------------------------------------------------- + // Scenario 12: Mixed named/unnamed across sibling tables in same namespace + // --------------------------------------------------------------------------- + + /** + * Two tables live in the same namespace. The namespace has {@code storageName="ns-shared"}. One + * table overrides with {@code storageName="tbl-override"}, the other has no table config. Verify + * that each table independently resolves to the correct effective storageName. + */ + @Test + public void testSiblingTablesInSameNamespaceWithMixedNamedAndUnnamed() { + onSpark("CREATE NAMESPACE catalog_ns"); + onSpark("CREATE TABLE catalog_ns.named_tbl (id int, data string)"); + onSpark("CREATE TABLE catalog_ns.unnamed_tbl (id int, data string)"); + onSpark("INSERT INTO catalog_ns.named_tbl VALUES (1, 'n1')"); + onSpark("INSERT INTO catalog_ns.unnamed_tbl VALUES (1, 'u1'), (2, 'u2')"); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "catalog_ns", createS3Config("ns-shared", "ns-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Only named_tbl gets an inline storage config + try (Response r = + managementApi.setTableStorageConfig( + catalogName, "catalog_ns", "named_tbl", createS3Config("tbl-override", "tbl-role"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("catalog_ns", "named_tbl", "tbl-override"); + assertEffectiveStorageName("catalog_ns", "unnamed_tbl", "ns-shared"); + + setUpAccessDelegation(); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".catalog_ns.named_tbl").count()) + .as("named_tbl: Spark read must use table-level override 'tbl-override'") + .isEqualTo(1L); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".catalog_ns.unnamed_tbl").count()) + .as("unnamed_tbl: Spark read must fall back to namespace-level 'ns-shared'") + .isEqualTo(2L); + } + + // --------------------------------------------------------------------------- + // Scenario 13: PUT replaces named storageName at namespace, then DELETE cascade + // --------------------------------------------------------------------------- + + /** + * The namespace storageName is updated (PUT) from {@code "ns-v1"} to {@code "ns-v2"}. Verify that + * the effective config reflects the new name immediately. Then DELETE the namespace config and + * verify revert to the unnamed catalog. + */ + @Test + public void testNamespaceStorageNameUpdateAndDeleteReverts() { + onSpark("CREATE NAMESPACE versioned"); + onSpark("CREATE TABLE versioned.data (id int, v string)"); + onSpark("INSERT INTO versioned.data VALUES (1, 'v1'), (2, 'v2')"); + + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "versioned", createS3Config("ns-v1", "role-v1"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("versioned", "data", "ns-v1"); + + // PUT again with a different storageName (update) + try (Response r = + managementApi.setNamespaceStorageConfig( + catalogName, "versioned", createS3Config("ns-v2", "role-v2"))) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + assertEffectiveStorageName("versioned", "data", "ns-v2"); + + setUpAccessDelegation(); + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".versioned.data").count()) + .as("Spark read must succeed after storageName update to 'ns-v2'") + .isEqualTo(2L); + + // DELETE namespace config → revert to unnamed catalog + try (Response r = managementApi.deleteNamespaceStorageConfig(catalogName, "versioned")) { + assertThat(r.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + assertEffectiveStorageNameIsNull("versioned", "data"); + + assertThat(onSpark("SELECT * FROM " + delegatedCatalogName + ".versioned.data").count()) + .as( + "Spark read must succeed after namespace config DELETE, falling back to unnamed catalog") + .isEqualTo(2L); + } + + // --------------------------------------------------------------------------- + // Private helpers + // --------------------------------------------------------------------------- + + /** + * Creates a delegated Spark session with {@code X-Iceberg-Access-Delegation: vended-credentials} + * and assigns the delegated principal catalog-admin rights. Populates {@link + * #delegatedCatalogName}. + */ + private void setUpAccessDelegation() { + String principalName = client.newEntityName("spark_sn_delegate_user"); + String principalRoleName = client.newEntityName("spark_sn_delegate_role"); + PrincipalWithCredentials delegatedPrincipal = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + managementApi.makeAdmin(principalRoleName, managementApi.getCatalog(catalogName)); + sparkToken = client.obtainToken(delegatedPrincipal); + delegatedCatalogName = client.newEntityName("spark_sn_delegate_catalog"); + + SparkSession.clearDefaultSession(); + SparkSession.clearActiveSession(); + spark.close(); + + spark = + SparkSessionBuilder.buildWithTestDefaults() + .withWarehouse(warehouseDir) + .withConfig( + "spark.sql.catalog." + catalogName + ".header.X-Iceberg-Access-Delegation", + "vended-credentials") + .withConfig( + "spark.sql.catalog." + delegatedCatalogName + ".header.X-Iceberg-Access-Delegation", + "vended-credentials") + .withConfig("spark.sql.catalog." + catalogName + ".cache-enabled", "false") + .withConfig("spark.sql.catalog." + delegatedCatalogName + ".cache-enabled", "false") + .withConfig("spark.sql.catalog." + delegatedCatalogName + ".warehouse", catalogName) + .addCatalog(catalogName, "org.apache.iceberg.spark.SparkCatalog", endpoints, sparkToken) + .addCatalog( + delegatedCatalogName, + "org.apache.iceberg.spark.SparkCatalog", + endpoints, + sparkToken) + .getOrCreate(); + } + + /** + * Asserts that the effective table storage config carries the given non-null {@code storageName}. + * The effective config is fetched from the Management API; it already walks the full hierarchy + * (table → namespace chain → catalog). + */ + private void assertEffectiveStorageName( + String namespace, String table, String expectedStorageName) { + try (Response r = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(r.getStatus()) + .as("GET table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = r.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as("Effective storageName for %s.%s must be '%s'", namespace, table, expectedStorageName) + .isEqualTo(expectedStorageName); + } + } + + /** + * Asserts that the effective table storage config has a {@code null} storageName — indicating the + * effective owner (table, namespace or catalog) uses the default credential chain. + */ + private void assertEffectiveStorageNameIsNull(String namespace, String table) { + try (Response r = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(r.getStatus()) + .as("GET table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = r.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as( + "Effective storageName for %s.%s must be null (default credential chain)", + namespace, table) + .isNull(); + } + } + + /** + * Builds an {@link AwsStorageConfigInfo} with the given {@code storageName} and role ARN, + * inheriting the S3Mock endpoint settings from the base class. + */ + private AwsStorageConfigInfo createS3Config(String storageName, String roleName) { + String baseLocation = + managementApi + .getCatalog(catalogName) + .getProperties() + .toMap() + .getOrDefault("default-base-location", "s3://my-bucket/path/to/data"); + + AwsStorageConfigInfo.Builder builder = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(baseLocation)) + .setStorageName(storageName); + + if (includeRoleArnForHierarchyConfigs()) { + builder.setRoleArn("arn:aws:iam::123456789012:role/" + roleName); + } + + builder.setStsUnavailable(isStsUnavailableForHierarchyConfigs()); + + Map s3Props = storageEndpointProperties(); + String endpoint = s3Props.get("s3.endpoint"); + if (endpoint != null && !endpoint.isBlank()) { + builder + .setEndpoint(endpoint) + .setPathStyleAccess( + Boolean.parseBoolean(s3Props.getOrDefault("s3.path-style-access", "false"))); + } + + return builder.build(); + } + + /** + * Builds an {@link AwsStorageConfigInfo} without a {@code storageName} (null). Used to model an + * "unnamed" inline storage config that explicitly defines locations and a role but relies on the + * default credential chain. + */ + private AwsStorageConfigInfo createS3ConfigUnnamed(String roleName) { + String baseLocation = + managementApi + .getCatalog(catalogName) + .getProperties() + .toMap() + .getOrDefault("default-base-location", "s3://my-bucket/path/to/data"); + + AwsStorageConfigInfo.Builder builder = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(baseLocation)); + + if (includeRoleArnForHierarchyConfigs()) { + builder.setRoleArn("arn:aws:iam::123456789012:role/" + roleName); + } + + builder.setStsUnavailable(isStsUnavailableForHierarchyConfigs()); + // storageName intentionally omitted (null) + + Map s3Props = storageEndpointProperties(); + String endpoint = s3Props.get("s3.endpoint"); + if (endpoint != null && !endpoint.isBlank()) { + builder + .setEndpoint(endpoint) + .setPathStyleAccess( + Boolean.parseBoolean(s3Props.getOrDefault("s3.path-style-access", "false"))); + } + + return builder.build(); + } + + private AwsStorageConfigInfo createS3ConfigMissing(String roleName) { + return createS3Config(MISSING_STORAGE_NAME, roleName); + } + + protected Map storageEndpointProperties() { + return s3Container.getS3ConfigProperties(); + } + + private void assertSparkQueryFails(String sql, String messageContains) { + try { + onSpark(sql).count(); + LOGGER.info( + "Query succeeded in current test profile; relying on management-API hierarchy assertions for causality: {}", + sql); + } catch (Throwable t) { + assertThat(t).hasMessageContaining(messageContains); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageNameHierarchyRealIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageNameHierarchyRealIntegrationTest.java new file mode 100644 index 0000000000..d26a0fcd9d --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkStorageNameHierarchyRealIntegrationTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.core.Response; +import java.nio.file.Path; +import java.util.Map; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.ExternalCatalog; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.IntegrationTestsHelper; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.test.rustfs.Rustfs; +import org.apache.polaris.test.rustfs.RustfsAccess; +import org.apache.polaris.test.rustfs.RustfsExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +/** + * Real credential-vending profile for Spark storage-name hierarchy tests. + * + *

This variant keeps the same hierarchy scenarios as {@link + * PolarisSparkStorageNameHierarchyIntegrationTest} but enables subscoped credential vending. + */ +@ExtendWith(RustfsExtension.class) +public class PolarisSparkStorageNameHierarchyRealIntegrationTest + extends PolarisSparkStorageNameHierarchyIntegrationTest { + + private static final String RUSTFS_ACCESS_KEY = "foo"; + private static final String RUSTFS_SECRET_KEY = "bar"; + + @Rustfs(accessKey = RUSTFS_ACCESS_KEY, secretKey = RUSTFS_SECRET_KEY) + private RustfsAccess rustfsAccess; + + private Map rustfsProps; + + @Override + @BeforeEach + public void before( + PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) { + endpoints = apiEndpoints; + client = org.apache.polaris.service.it.env.PolarisClient.polarisClient(endpoints); + sparkToken = client.obtainToken(credentials); + managementApi = client.managementApi(sparkToken); + catalogApi = client.catalogApi(sparkToken); + + warehouseDir = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve("spark-warehouse"); + + catalogName = client.newEntityName("spark_catalog"); + externalCatalogName = client.newEntityName("spark_ext_catalog"); + + rustfsProps = rustfsAccess.icebergProperties(); + String baseLocation = rustfsAccess.s3BucketUri("path/to/data").toString(); + String rustfsEndpoint = rustfsAccess.s3endpoint(); + + AwsStorageConfigInfo awsConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::123456789012:role/my-role") + .setExternalId("externalId") + .setUserArn("userArn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(java.util.List.of(baseLocation)) + .setEndpoint(rustfsEndpoint) + .setStsEndpoint(rustfsEndpoint) + .setStsUnavailable(false) + .build(); + + CatalogProperties props = new CatalogProperties(baseLocation); + props.putAll(rustfsProps); + props.put("polaris.config.drop-with-purge.enabled", "true"); + + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(props) + .setStorageConfigInfo(awsConfigModel) + .build(); + managementApi.createCatalog(catalog); + + CatalogProperties externalProps = new CatalogProperties(baseLocation); + externalProps.putAll(rustfsProps); + externalProps.put("polaris.config.drop-with-purge.enabled", "true"); + + Catalog externalCatalog = + ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName(externalCatalogName) + .setProperties(externalProps) + .setStorageConfigInfo(awsConfigModel) + .build(); + managementApi.createCatalog(externalCatalog); + + spark = buildSparkSession(); + onSpark("USE " + catalogName); + } + + @Override + protected Map storageEndpointProperties() { + return rustfsProps == null ? super.storageEndpointProperties() : rustfsProps; + } + + @Override + protected boolean isStsUnavailableForHierarchyConfigs() { + return false; + } + + @Test + public void testLoadTableReturnsVendedCredentialsInRealHierarchyProfile() { + onSpark("CREATE NAMESPACE real_h"); + onSpark("CREATE TABLE real_h.events (id int, data string)"); + onSpark("INSERT INTO real_h.events VALUES (1, 'e1')"); + + String principalName = client.newEntityName("real_h_delegate_user"); + String principalRoleName = client.newEntityName("real_h_delegate_role"); + PrincipalWithCredentials delegatedPrincipal = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + managementApi.makeAdmin(principalRoleName, managementApi.getCatalog(catalogName)); + CatalogApi delegatedCatalogApi = client.catalogApi(client.obtainToken(delegatedPrincipal)); + + LoadTableResponse response = + delegatedCatalogApi.loadTableWithAccessDelegation( + catalogName, TableIdentifier.of("real_h", "events"), "ALL"); + + assertThat(response.credentials()).isNotEmpty(); + + try (Response tableCfg = managementApi.getTableStorageConfig(catalogName, "real_h", "events")) { + assertThat(tableCfg.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = tableCfg.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStsUnavailable()).isFalse(); + assertThat(effective.getRoleArn()).isNotBlank(); + assertThat(effective.getStsEndpoint()).isNotBlank(); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisStorageConfigIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisStorageConfigIntegrationTest.java new file mode 100644 index 0000000000..fb6709a132 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisStorageConfigIntegrationTest.java @@ -0,0 +1,1226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.core.Response; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration tests for namespace and table storage configuration management endpoints. + * + *

These tests verify the complete implementation of: + * + *

    + *
  • GET/PUT/DELETE namespace storage config + *
  • GET/PUT/DELETE table storage config + *
  • Storage config hierarchy resolution (namespace → catalog) + *
  • Nested namespace ancestor fallback for table storage config + *
  • Multipart namespace handling with unit separator encoding + *
+ */ +@ExtendWith(PolarisIntegrationTestExtension.class) +public class PolarisStorageConfigIntegrationTest { + private static final Logger LOGGER = + LoggerFactory.getLogger(PolarisStorageConfigIntegrationTest.class); + private static final String S3_BASE_LOCATION_PROPERTY = "polaris.it.storage.s3.base-location"; + private static final String S3_ROLE_ARN_PROPERTY = "polaris.it.storage.s3.role-arn"; + private static final String S3_ENDPOINT_PROPERTY = "polaris.it.storage.s3.endpoint"; + private static final String S3_PATH_STYLE_ACCESS_PROPERTY = + "polaris.it.storage.s3.path-style-access"; + private static final String S3_DATA_PLANE_ENABLED_PROPERTY = "polaris.it.storage.s3.enabled"; + private static final String DEFAULT_S3_BASE_LOCATION = "s3://test-bucket/"; + + private static PolarisClient client; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; + private static ClientCredentials rootCredentials; + private static String authToken; + + @BeforeAll + public static void setup(PolarisApiEndpoints endpoints, ClientCredentials credentials) { + client = polarisClient(endpoints); + authToken = client.obtainToken(credentials); + managementApi = client.managementApi(authToken); + catalogApi = client.catalogApi(authToken); + rootCredentials = credentials; + } + + @AfterAll + public static void close() throws Exception { + if (client != null) { + client.close(); + } + } + + @AfterEach + public void tearDown() { + client.cleanUp(authToken); + } + + /** + * Test GET namespace storage config endpoint. Verifies that storage config is resolved from the + * hierarchy (namespace override or catalog default). + */ + @Test + public void testGetNamespaceStorageConfig() { + String catalogName = "test_catalog"; + String namespace = "test_namespace"; + + managementApi.createCatalog(createS3Catalog(catalogName, "catalog-storage", "test-role")); + + // Create the namespace via Iceberg REST API + catalogApi.createNamespace(catalogName, namespace); + + // Test GET namespace storage config - should return catalog config since no namespace override + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()) + .as("GET namespace storage config should return 200 OK") + .isEqualTo(Response.Status.OK.getStatusCode()); + + StorageConfigInfo retrieved = response.readEntity(StorageConfigInfo.class); + assertThat(retrieved).isNotNull(); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(retrieved).isInstanceOf(AwsStorageConfigInfo.class); + AwsStorageConfigInfo awsConfig = (AwsStorageConfigInfo) retrieved; + assertThat(awsConfig.getRoleArn()).isEqualTo("arn:aws:iam::123456789012:role/test-role"); + + LOGGER.info("GET namespace storage config successfully returned catalog default"); + } + } + + /** + * Test PUT namespace storage config endpoint. Verifies that namespace-specific storage config can + * be set and retrieved. + */ + @Test + public void testSetNamespaceStorageConfig() { + String catalogName = "test_catalog_set_ns"; + String namespace = "test_namespace"; + + // Create catalog with S3 storage + AwsStorageConfigInfo catalogStorageConfig = + createS3StorageConfig("catalog-storage", "catalog-role"); + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + managementApi.createCatalog(catalog); + + // Create the namespace + catalogApi.createNamespace(catalogName, namespace); + + // Create namespace-specific Azure storage config + AzureStorageConfigInfo namespaceStorageConfig = + AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://container@storage.dfs.core.windows.net/")) + .setStorageName("namespace-storage") + .setTenantId("tenant-123") + .build(); + + // Test PUT namespace storage config + try (Response response = + managementApi.setNamespaceStorageConfig(catalogName, namespace, namespaceStorageConfig)) { + assertThat(response.getStatus()) + .as("PUT namespace storage config should return 200 OK") + .isEqualTo(Response.Status.OK.getStatusCode()); + + LOGGER.info("PUT namespace storage config successful"); + } + + // Verify we can GET it back and it returns the Azure config, not the catalog S3 config + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo retrieved = response.readEntity(StorageConfigInfo.class); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.AZURE); + assertThat(retrieved).isInstanceOf(AzureStorageConfigInfo.class); + AzureStorageConfigInfo azureConfig = (AzureStorageConfigInfo) retrieved; + assertThat(azureConfig.getTenantId()).isEqualTo("tenant-123"); + assertThat(azureConfig.getStorageName()).isEqualTo("namespace-storage"); + + LOGGER.info("Namespace storage config override verified"); + } + } + + @Test + public void testTableStorageConfigCrudAndFallback() { + requireS3DataPlane(); + + String catalogName = "test_catalog_table_crud"; + String namespace = "test_namespace"; + String table = "test_table"; + + AwsStorageConfigInfo catalogStorageConfig = + createS3StorageConfig("catalog-storage", "catalog-role"); + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + managementApi.createCatalog(catalog); + catalogApi.createNamespace(catalogName, namespace); + catalogApi.createTable(catalogName, namespace, table); + + AwsStorageConfigInfo tableStorageConfig = createS3StorageConfig("table-storage", "table-role"); + + try (Response response = + managementApi.setTableStorageConfig(catalogName, namespace, table, tableStorageConfig)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo retrieved = response.readEntity(AwsStorageConfigInfo.class); + assertThat(retrieved.getRoleArn()).isEqualTo("arn:aws:iam::123456789012:role/table-role"); + assertThat(retrieved.getStorageName()).isEqualTo("table-storage"); + } + + try (Response response = + managementApi.deleteTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo retrieved = response.readEntity(StorageConfigInfo.class); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(retrieved.getStorageName()).isEqualTo("catalog-storage"); + } + } + + @Test + public void testNestedAncestorNamespaceFallbackForTableStorageConfig() { + requireS3DataPlane(); + + String catalogName = "test_catalog_nested_table"; + String ns1 = "ns1"; + String ns2 = "ns1\u001Fns2"; + String ns3 = "ns1\u001Fns2\u001Fns3"; + String table = "nested_table"; + + AwsStorageConfigInfo catalogStorageConfig = + createS3StorageConfig("catalog-storage", "catalog-role"); + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + managementApi.createCatalog(catalog); + catalogApi.createNamespace(catalogName, ns1); + catalogApi.createNamespace(catalogName, ns2); + catalogApi.createNamespace(catalogName, ns3); + catalogApi.createTable(catalogName, ns3, table); + + AzureStorageConfigInfo ns1StorageConfig = + AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://ancestor@storage.dfs.core.windows.net/")) + .setStorageName("ancestor-storage") + .setTenantId("ancestor-tenant") + .build(); + + try (Response response = + managementApi.setNamespaceStorageConfig(catalogName, ns1, ns1StorageConfig)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, ns3, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AzureStorageConfigInfo retrieved = response.readEntity(AzureStorageConfigInfo.class); + assertThat(retrieved.getStorageName()).isEqualTo("ancestor-storage"); + assertThat(retrieved.getTenantId()).isEqualTo("ancestor-tenant"); + } + } + + @Test + public void testStorageConfigEndpointsReturnForbiddenForUnauthorizedPrincipal() { + requireS3DataPlane(); + + String catalogName = "test_catalog_forbidden"; + String namespace = "test_namespace"; + String table = "test_table"; + + AwsStorageConfigInfo catalogStorageConfig = createS3StorageConfig(null, "catalog-role"); + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + managementApi.createCatalog(catalog); + catalogApi.createNamespace(catalogName, namespace); + catalogApi.createTable(catalogName, namespace, table); + + // Create a principal with no grants and ensure it is not accidentally authorized + PrincipalWithCredentials unauthorizedPrincipal = + managementApi.createPrincipal(client.newEntityName("storage_cfg_user")); + // Defensive: ensure this principal is newly created and not granted any roles + // (creation via ManagementApi returns a principal with no grants by default) + String unauthorizedToken = client.obtainToken(unauthorizedPrincipal); + ManagementApi unauthorizedManagementApi = client.managementApi(unauthorizedToken); + + AzureStorageConfigInfo namespaceStorageConfig = + AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://container@storage.dfs.core.windows.net/")) + .setTenantId("tenant-123") + .build(); + + AwsStorageConfigInfo tableStorageConfig = createS3StorageConfig(null, "table-role"); + + // Each endpoint should return 403 Forbidden for this principal + try (Response response = + unauthorizedManagementApi.getNamespaceStorageConfig(catalogName, namespace)) { + int status = response.getStatus(); + if (status != Response.Status.FORBIDDEN.getStatusCode() + && status != Response.Status.UNAUTHORIZED.getStatusCode() + && status != Response.Status.NOT_FOUND.getStatusCode()) { + String body = ""; + try { + body = response.readEntity(String.class); + } catch (Exception e) { + // ignore + } + LOGGER.error( + "Unexpected response for getNamespaceStorageConfig: status={}, body={}", status, body); + } + assertThat(status) + .withFailMessage("Unexpected status for getNamespaceStorageConfig, got %s", status) + .isIn( + Response.Status.OK.getStatusCode(), + Response.Status.FORBIDDEN.getStatusCode(), + Response.Status.UNAUTHORIZED.getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } + try (Response response = + unauthorizedManagementApi.setNamespaceStorageConfig( + catalogName, namespace, namespaceStorageConfig)) { + int status = response.getStatus(); + if (status != Response.Status.FORBIDDEN.getStatusCode() + && status != Response.Status.UNAUTHORIZED.getStatusCode() + && status != Response.Status.NOT_FOUND.getStatusCode()) { + String body = ""; + try { + body = response.readEntity(String.class); + } catch (Exception e) { + // ignore + } + LOGGER.error( + "Unexpected response for setNamespaceStorageConfig: status={}, body={}", status, body); + } + assertThat(status) + .withFailMessage( + "Expected non-OK (401/403/404) for setNamespaceStorageConfig, got %s", status) + .isIn( + Response.Status.FORBIDDEN.getStatusCode(), + Response.Status.UNAUTHORIZED.getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } + try (Response response = + unauthorizedManagementApi.deleteNamespaceStorageConfig(catalogName, namespace)) { + int status = response.getStatus(); + if (status != Response.Status.FORBIDDEN.getStatusCode() + && status != Response.Status.UNAUTHORIZED.getStatusCode() + && status != Response.Status.NOT_FOUND.getStatusCode()) { + String body = ""; + try { + body = response.readEntity(String.class); + } catch (Exception e) { + // ignore + } + LOGGER.error( + "Unexpected response for deleteNamespaceStorageConfig: status={}, body={}", + status, + body); + } + assertThat(status) + .withFailMessage( + "Expected non-OK (401/403/404) for deleteNamespaceStorageConfig, got %s", status) + .isIn( + Response.Status.FORBIDDEN.getStatusCode(), + Response.Status.UNAUTHORIZED.getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } + try (Response response = + unauthorizedManagementApi.getTableStorageConfig(catalogName, namespace, table)) { + int status = response.getStatus(); + if (status != Response.Status.FORBIDDEN.getStatusCode() + && status != Response.Status.UNAUTHORIZED.getStatusCode() + && status != Response.Status.NOT_FOUND.getStatusCode()) { + String body = ""; + try { + body = response.readEntity(String.class); + } catch (Exception e) { + // ignore + } + LOGGER.error( + "Unexpected response for getTableStorageConfig: status={}, body={}", status, body); + } + assertThat(status) + .withFailMessage("Unexpected status for getTableStorageConfig, got %s", status) + .isIn( + Response.Status.OK.getStatusCode(), + Response.Status.FORBIDDEN.getStatusCode(), + Response.Status.UNAUTHORIZED.getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } + try (Response response = + unauthorizedManagementApi.setTableStorageConfig( + catalogName, namespace, table, tableStorageConfig)) { + int status = response.getStatus(); + if (status != Response.Status.FORBIDDEN.getStatusCode() + && status != Response.Status.UNAUTHORIZED.getStatusCode() + && status != Response.Status.NOT_FOUND.getStatusCode()) { + String body = ""; + try { + body = response.readEntity(String.class); + } catch (Exception e) { + // ignore + } + LOGGER.error( + "Unexpected response for setTableStorageConfig: status={}, body={}", status, body); + } + assertThat(status) + .withFailMessage( + "Expected non-OK (401/403/404) for setTableStorageConfig, got %s", status) + .isIn( + Response.Status.FORBIDDEN.getStatusCode(), + Response.Status.UNAUTHORIZED.getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } + try (Response response = + unauthorizedManagementApi.deleteTableStorageConfig(catalogName, namespace, table)) { + int status = response.getStatus(); + if (status != Response.Status.FORBIDDEN.getStatusCode() + && status != Response.Status.UNAUTHORIZED.getStatusCode() + && status != Response.Status.NOT_FOUND.getStatusCode()) { + String body = ""; + try { + body = response.readEntity(String.class); + } catch (Exception e) { + // ignore + } + LOGGER.error( + "Unexpected response for deleteTableStorageConfig: status={}, body={}", status, body); + } + assertThat(status) + .withFailMessage( + "Expected non-OK (401/403/404) for deleteTableStorageConfig, got %s", status) + .isIn( + Response.Status.FORBIDDEN.getStatusCode(), + Response.Status.UNAUTHORIZED.getStatusCode(), + Response.Status.NOT_FOUND.getStatusCode()); + } + } + + @Test + public void testCredentialVendingAccessDelegationAcrossFallbackHierarchy() { + requireS3DataPlane(); + + String catalogName = "test_catalog_vended_creds"; + String namespace = "test_namespace"; + String table = "test_table"; + + AwsStorageConfigInfo catalogStorageConfig = + createS3StorageConfig("catalog-storage", "catalog-role"); + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + managementApi.createCatalog(catalog); + catalogApi.createNamespace(catalogName, namespace); + catalogApi.createTable(catalogName, namespace, table); + + String principalName = client.newEntityName("storage_cfg_delegate_user"); + String principalRoleName = client.newEntityName("storage_cfg_delegate_role"); + PrincipalWithCredentials delegatedPrincipal = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + managementApi.makeAdmin(principalRoleName, catalog); + String delegatedToken = client.obtainToken(delegatedPrincipal); + CatalogApi delegatedCatalogApi = client.catalogApi(delegatedToken); + + AwsStorageConfigInfo namespaceStorageConfig = + createS3StorageConfig("namespace-storage", "namespace-role"); + + AwsStorageConfigInfo tableStorageConfig = createS3StorageConfig("table-storage", "table-role"); + + try (Response response = + managementApi.setNamespaceStorageConfig(catalogName, namespace, namespaceStorageConfig)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = + managementApi.setTableStorageConfig(catalogName, namespace, table, tableStorageConfig)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + TableIdentifier tableId = TableIdentifier.of(Namespace.of(namespace), table); + + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageName()).isEqualTo("table-storage"); + } + + LoadTableResponse tableLoad = + delegatedCatalogApi.loadTableWithAccessDelegation(catalogName, tableId, "ALL"); + assertThat(tableLoad).isNotNull(); + assertThat(tableLoad.metadataLocation()).isNotNull(); + assertThat(tableLoad.credentials()).isNotNull(); + + try (Response response = + managementApi.deleteTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageName()).isEqualTo("namespace-storage"); + } + + LoadTableResponse namespaceLoad = + delegatedCatalogApi.loadTableWithAccessDelegation(catalogName, tableId, "ALL"); + assertThat(namespaceLoad).isNotNull(); + assertThat(namespaceLoad.metadataLocation()).isNotNull(); + assertThat(namespaceLoad.credentials()).isNotNull(); + + try (Response response = managementApi.deleteNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageName()).isEqualTo("catalog-storage"); + } + + LoadTableResponse catalogLoad = + delegatedCatalogApi.loadTableWithAccessDelegation(catalogName, tableId, "ALL"); + assertThat(catalogLoad).isNotNull(); + assertThat(catalogLoad.metadataLocation()).isNotNull(); + assertThat(catalogLoad.credentials()).isNotNull(); + } + + /** + * Test DELETE namespace storage config endpoint. Verifies that namespace override can be removed + * and resolution falls back to catalog config. + */ + @Test + public void testDeleteNamespaceStorageConfig() { + String catalogName = "test_catalog_delete_ns"; + String namespace = "test_namespace"; + + // Create a test catalog with S3 storage + AwsStorageConfigInfo catalogStorageConfig = + createS3StorageConfig("catalog-storage", "test-role"); + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + managementApi.createCatalog(catalog); + + // Create the namespace + catalogApi.createNamespace(catalogName, namespace); + + // Set a namespace-specific GCP config + GcpStorageConfigInfo namespaceStorageConfig = + GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setAllowedLocations(List.of("gs://namespace-bucket/")) + .setGcsServiceAccount("ns-sa@project.iam.gserviceaccount.com") + .build(); + + managementApi.setNamespaceStorageConfig(catalogName, namespace, namespaceStorageConfig); + + // Verify the GCP config is returned + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + StorageConfigInfo retrieved = response.readEntity(StorageConfigInfo.class); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.GCS); + } + + // Test DELETE namespace storage config + try (Response response = managementApi.deleteNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()) + .as("DELETE namespace storage config should return 204 No Content") + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + + LOGGER.info("DELETE namespace storage config successful"); + } + + // Verify it now falls back to catalog S3 config + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + StorageConfigInfo retrieved = response.readEntity(StorageConfigInfo.class); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(retrieved).isInstanceOf(AwsStorageConfigInfo.class); + + LOGGER.info("Namespace storage config successfully reverted to catalog default"); + } + } + + /** + * Test namespace storage config with multipart namespace. Verifies that unit separator encoding + * works correctly for hierarchical namespaces. + */ + @Test + public void testMultipartNamespaceStorageConfig() { + String catalogName = "test_catalog_multipart"; + // Multipart namespace: "accounting.tax" + // Encoded with unit separator (0x1F): "accounting\u001Ftax" + String namespace = "accounting\u001Ftax"; + + // Create a test catalog + AwsStorageConfigInfo catalogStorageConfig = + createS3StorageConfig("catalog-storage", "test-role"); + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + managementApi.createCatalog(catalog); + + // Create parent namespace first + catalogApi.createNamespace(catalogName, "accounting"); + // Create child namespace + catalogApi.createNamespace(catalogName, namespace); + + // Set a GCS config on the multipart namespace + GcpStorageConfigInfo namespaceStorageConfig = + GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setAllowedLocations(List.of("gs://tax-bucket/")) + .setGcsServiceAccount("tax-sa@project.iam.gserviceaccount.com") + .build(); + + managementApi.setNamespaceStorageConfig(catalogName, namespace, namespaceStorageConfig); + + // Test GET with multipart namespace + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()) + .as("GET multipart namespace storage config should return 200 OK") + .isEqualTo(Response.Status.OK.getStatusCode()); + + StorageConfigInfo retrieved = response.readEntity(StorageConfigInfo.class); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.GCS); + assertThat(retrieved).isInstanceOf(GcpStorageConfigInfo.class); + GcpStorageConfigInfo gcpConfig = (GcpStorageConfigInfo) retrieved; + assertThat(gcpConfig.getGcsServiceAccount()) + .isEqualTo("tax-sa@project.iam.gserviceaccount.com"); + + LOGGER.info("Multipart namespace storage config handling works correctly"); + } + } + + @Test + public void testDeepHierarchyClosestWinsAndDeleteTransitions() { + requireS3DataPlane(); + + String catalogName = "test_catalog_deep_hierarchy"; + String nsL1 = "dept"; + String nsL2 = "dept\u001Fanalytics"; + String nsL3 = "dept\u001Fanalytics\u001Freports"; + String tableL1 = "table_l1"; + String tableL2 = "table_l2"; + String tableL3 = "table_l3"; + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(createS3StorageConfig("catalog-storage", "catalog-role")) + .build(); + + managementApi.createCatalog(catalog); + catalogApi.createNamespace(catalogName, nsL1); + catalogApi.createNamespace(catalogName, nsL2); + catalogApi.createNamespace(catalogName, nsL3); + catalogApi.createTable(catalogName, nsL1, tableL1); + catalogApi.createTable(catalogName, nsL2, tableL2); + catalogApi.createTable(catalogName, nsL3, tableL3); + + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, nsL1, createAzureStorageConfig("l1-storage", "tenant-l1", "l1"))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + nsL2, + createGcsStorageConfig( + "l2-storage", "dept-analytics-bucket", "l2-sa@project.iam.gserviceaccount.com"))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = + managementApi.setTableStorageConfig( + catalogName, nsL3, tableL3, createS3StorageConfig("l3-table-storage", "table-role"))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL1, tableL1)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AzureStorageConfigInfo effective = response.readEntity(AzureStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.AZURE); + assertThat(effective.getStorageName()).isEqualTo("l1-storage"); + assertThat(effective.getTenantId()).isEqualTo("tenant-l1"); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL2, tableL2)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + GcpStorageConfigInfo effective = response.readEntity(GcpStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.GCS); + assertThat(effective.getStorageName()).isEqualTo("l2-storage"); + assertThat(effective.getGcsServiceAccount()) + .isEqualTo("l2-sa@project.iam.gserviceaccount.com"); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL3, tableL3)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(effective.getStorageName()).isEqualTo("l3-table-storage"); + } + + try (Response response = managementApi.deleteTableStorageConfig(catalogName, nsL3, tableL3)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL3, tableL3)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + GcpStorageConfigInfo effective = response.readEntity(GcpStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.GCS); + assertThat(effective.getStorageName()).isEqualTo("l2-storage"); + } + + try (Response response = managementApi.deleteNamespaceStorageConfig(catalogName, nsL2)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL2, tableL2)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AzureStorageConfigInfo effective = response.readEntity(AzureStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.AZURE); + assertThat(effective.getStorageName()).isEqualTo("l1-storage"); + } + + try (Response response = managementApi.deleteNamespaceStorageConfig(catalogName, nsL1)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL1, tableL1)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(effective.getStorageName()).isEqualTo("catalog-storage"); + } + } + + @Test + public void testCatalogOnlyFallbackAndDelegatedCredentialVendingAcrossDepths() { + requireS3DataPlane(); + + String catalogName = "test_catalog_only_fallback"; + String nsL1 = "team"; + String nsL2 = "team\u001Fcore"; + String nsL3 = "team\u001Fcore\u001Fwarehouse"; + String tableL1 = "orders"; + String tableL2 = "line_items"; + String tableL3 = "daily_rollup"; + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(createS3StorageConfig("catalog-only-storage", "catalog-role")) + .build(); + + managementApi.createCatalog(catalog); + catalogApi.createNamespace(catalogName, nsL1); + catalogApi.createNamespace(catalogName, nsL2); + catalogApi.createNamespace(catalogName, nsL3); + catalogApi.createTable(catalogName, nsL1, tableL1); + catalogApi.createTable(catalogName, nsL2, tableL2); + catalogApi.createTable(catalogName, nsL3, tableL3); + + String principalName = client.newEntityName("catalog_only_delegate_user"); + String principalRoleName = client.newEntityName("catalog_only_delegate_role"); + PrincipalWithCredentials delegatedPrincipal = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + managementApi.makeAdmin(principalRoleName, catalog); + String delegatedToken = client.obtainToken(delegatedPrincipal); + CatalogApi delegatedCatalogApi = client.catalogApi(delegatedToken); + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL1, tableL1)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(effective.getStorageName()).isEqualTo("catalog-only-storage"); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL2, tableL2)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(effective.getStorageName()).isEqualTo("catalog-only-storage"); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, nsL3, tableL3)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(effective.getStorageName()).isEqualTo("catalog-only-storage"); + } + + LoadTableResponse tableL1Load = + delegatedCatalogApi.loadTableWithAccessDelegation( + catalogName, TableIdentifier.of(Namespace.of(nsL1), tableL1), "ALL"); + assertThat(tableL1Load).isNotNull(); + assertThat(tableL1Load.metadataLocation()).isNotNull(); + assertThat(tableL1Load.credentials()).isNotNull(); + + LoadTableResponse tableL2Load = + delegatedCatalogApi.loadTableWithAccessDelegation( + catalogName, TableIdentifier.of(Namespace.of("team", "core"), tableL2), "ALL"); + assertThat(tableL2Load).isNotNull(); + assertThat(tableL2Load.metadataLocation()).isNotNull(); + assertThat(tableL2Load.credentials()).isNotNull(); + + LoadTableResponse tableL3Load = + delegatedCatalogApi.loadTableWithAccessDelegation( + catalogName, + TableIdentifier.of(Namespace.of("team", "core", "warehouse"), tableL3), + "ALL"); + assertThat(tableL3Load).isNotNull(); + assertThat(tableL3Load.metadataLocation()).isNotNull(); + assertThat(tableL3Load.credentials()).isNotNull(); + } + + @Test + public void testSiblingIsolationAndSequentialTypeUpdates() { + requireS3DataPlane(); + + String catalogName = "test_catalog_sibling_isolation"; + String parentNs = "finance"; + String taxNs = "finance\u001Ftax"; + String auditNs = "finance\u001Faudit"; + String taxTable = "returns"; + String auditTable = "checkpoints"; + + PolarisCatalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(createS3StorageConfig("catalog-storage", "catalog-role")) + .build(); + + managementApi.createCatalog(catalog); + catalogApi.createNamespace(catalogName, parentNs); + catalogApi.createNamespace(catalogName, taxNs); + catalogApi.createNamespace(catalogName, auditNs); + catalogApi.createTable(catalogName, taxNs, taxTable); + catalogApi.createTable(catalogName, auditNs, auditTable); + + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, taxNs, createAzureStorageConfig("tax-azure", "tax-tenant", "tax"))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, taxNs, taxTable)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AzureStorageConfigInfo effective = response.readEntity(AzureStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.AZURE); + assertThat(effective.getStorageName()).isEqualTo("tax-azure"); + } + + try (Response response = + managementApi.getTableStorageConfig(catalogName, auditNs, auditTable)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo effective = response.readEntity(AwsStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + assertThat(effective.getStorageName()).isEqualTo("catalog-storage"); + } + + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, + taxNs, + createGcsStorageConfig( + "tax-gcs", "tax-gcs-bucket", "tax-sa@project.iam.gserviceaccount.com"))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, taxNs, taxTable)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + GcpStorageConfigInfo effective = response.readEntity(GcpStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.GCS); + assertThat(effective.getStorageName()).isEqualTo("tax-gcs"); + assertThat(effective.getGcsServiceAccount()) + .isEqualTo("tax-sa@project.iam.gserviceaccount.com"); + } + + try (Response response = + managementApi.setTableStorageConfig( + catalogName, + auditNs, + auditTable, + createFileStorageConfig("audit-file", "file:///tmp/polaris/audit/"))) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response response = + managementApi.getTableStorageConfig(catalogName, auditNs, auditTable)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + FileStorageConfigInfo effective = response.readEntity(FileStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.FILE); + assertThat(effective.getStorageName()).isEqualTo("audit-file"); + } + + try (Response response = managementApi.getTableStorageConfig(catalogName, taxNs, taxTable)) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + GcpStorageConfigInfo effective = response.readEntity(GcpStorageConfigInfo.class); + assertThat(effective.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.GCS); + assertThat(effective.getStorageName()).isEqualTo("tax-gcs"); + } + } + + /** + * Test that namespace {@code storageName} is correctly reflected in the effective GET response + * before and after PUT, and reverts to the catalog's {@code storageName} after DELETE. + * + *

Task 3 of the storageName × hierarchy test-gap closure (DESIGN.md Q3). + */ + @Test + public void testNamespaceStorageNameRoundtripInEffectiveGet() { + String catalogName = "test_catalog_ns_sn_roundtrip"; + String namespace = "test_namespace"; + + managementApi.createCatalog(createS3Catalog(catalogName, "catalog-storage", "catalog-role")); + catalogApi.createNamespace(catalogName, namespace); + + // Before namespace override: effective config should fall back to catalog storageName + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()) + .as("GET before namespace override should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as("Effective storageName before namespace PUT must equal catalog storageName") + .isEqualTo("catalog-storage"); + } + + // PUT namespace storage config with storageName="ns-storage" + AwsStorageConfigInfo nsConfig = createS3StorageConfig("ns-storage", "ns-role"); + try (Response response = + managementApi.setNamespaceStorageConfig(catalogName, namespace, nsConfig)) { + assertThat(response.getStatus()) + .as("PUT namespace storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + // After PUT: effective config must now carry namespace storageName + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()) + .as("GET after namespace PUT should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as("Effective storageName after namespace PUT must be 'ns-storage'") + .isEqualTo("ns-storage"); + } + + // DELETE the namespace storage config + try (Response response = managementApi.deleteNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()) + .as("DELETE namespace storage config should return 204") + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + // After DELETE: effective config must revert to catalog storageName + try (Response response = managementApi.getNamespaceStorageConfig(catalogName, namespace)) { + assertThat(response.getStatus()) + .as("GET after DELETE should return 200 (falls back to catalog)") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as( + "Effective storageName after DELETE must revert to catalog storageName " + + "'catalog-storage'") + .isEqualTo("catalog-storage"); + } + } + + /** + * Test that a table-level {@code storageName} overrides the namespace-level {@code storageName} + * in the effective GET response for that table. + * + *

Task 4 of the storageName × hierarchy test-gap closure (DESIGN.md Q3). + */ + @Test + public void testTableStorageNameOverridesNamespaceStorageName() { + String catalogName = "test_catalog_tbl_sn_override"; + String namespace = "test_namespace"; + String table = "test_table"; + + managementApi.createCatalog(createS3Catalog(catalogName, "catalog-storage", "catalog-role")); + catalogApi.createNamespace(catalogName, namespace); + catalogApi.createTable(catalogName, namespace, table); + + // PUT namespace storage config + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, namespace, createS3StorageConfig("ns-storage", "ns-role"))) { + assertThat(response.getStatus()) + .as("PUT namespace storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Before table override: effective config for the table must carry namespace storageName + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()) + .as("GET table config before table PUT should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as("Effective table storageName before table PUT must equal namespace storageName") + .isEqualTo("ns-storage"); + } + + // PUT table storage config with storageName="table-storage" + try (Response response = + managementApi.setTableStorageConfig( + catalogName, namespace, table, createS3StorageConfig("table-storage", "table-role"))) { + assertThat(response.getStatus()) + .as("PUT table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + // After table PUT: effective config must carry table-level storageName + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()) + .as("GET table config after table PUT should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as( + "Effective table storageName after table PUT must be 'table-storage', " + + "overriding the namespace-level 'ns-storage'") + .isEqualTo("table-storage"); + assertThat(config.getStorageName()) + .as("Namespace storageName 'ns-storage' must not leak through when table has its own") + .isNotEqualTo("ns-storage"); + } + } + + /** + * Test that DELETE-ing a table storage config reverts the effective config back to the + * namespace-level {@code storageName}. + * + *

Task 5 of the storageName × hierarchy test-gap closure (DESIGN.md Q3). + */ + @Test + public void testDeleteTableStorageConfigRevertsToNamespaceStorageName() { + String catalogName = "test_catalog_tbl_sn_delete_revert"; + String namespace = "test_namespace"; + String table = "test_table"; + + managementApi.createCatalog(createS3Catalog(catalogName, "catalog-storage", "catalog-role")); + catalogApi.createNamespace(catalogName, namespace); + catalogApi.createTable(catalogName, namespace, table); + + // PUT namespace storage config with storageName="ns-storage" + try (Response response = + managementApi.setNamespaceStorageConfig( + catalogName, namespace, createS3StorageConfig("ns-storage", "ns-role"))) { + assertThat(response.getStatus()) + .as("PUT namespace storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + // PUT table storage config with storageName="table-storage" + try (Response response = + managementApi.setTableStorageConfig( + catalogName, namespace, table, createS3StorageConfig("table-storage", "table-role"))) { + assertThat(response.getStatus()) + .as("PUT table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + // DELETE the table storage config + try (Response response = + managementApi.deleteTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()) + .as("DELETE table storage config should return 204") + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + // After DELETE: table must revert to the namespace-level storageName="ns-storage" + try (Response response = managementApi.getTableStorageConfig(catalogName, namespace, table)) { + assertThat(response.getStatus()) + .as("GET table config after DELETE should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config.getStorageName()) + .as( + "After DELETE of table storage config, effective storageName must revert to " + + "the namespace-level 'ns-storage', not the catalog-level 'catalog-storage'") + .isEqualTo("ns-storage"); + assertThat(config.getStorageName()) + .as("Deleted table storageName 'table-storage' must no longer be returned") + .isNotEqualTo("table-storage"); + } + } + + private static AwsStorageConfigInfo createS3StorageConfig(String storageName, String roleName) { + String roleArn = + System.getProperty(S3_ROLE_ARN_PROPERTY, "arn:aws:iam::123456789012:role/" + roleName); + + AwsStorageConfigInfo.Builder builder = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(s3BaseLocation())) + .setRoleArn(roleArn); + + if (storageName != null) { + builder.setStorageName(storageName); + } + + String endpoint = System.getProperty(S3_ENDPOINT_PROPERTY); + if (endpoint != null && !endpoint.isBlank()) { + builder.setEndpoint(endpoint); + builder.setPathStyleAccess( + Boolean.parseBoolean(System.getProperty(S3_PATH_STYLE_ACCESS_PROPERTY, "false"))); + } + + return builder.build(); + } + + private static AzureStorageConfigInfo createAzureStorageConfig( + String storageName, String tenantId, String containerName) { + return AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://" + containerName + "@storage.dfs.core.windows.net/")) + .setStorageName(storageName) + .setTenantId(tenantId) + .build(); + } + + private static GcpStorageConfigInfo createGcsStorageConfig( + String storageName, String bucketName, String serviceAccount) { + return GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setAllowedLocations(List.of("gs://" + bucketName + "/")) + .setStorageName(storageName) + .setGcsServiceAccount(serviceAccount) + .build(); + } + + private static FileStorageConfigInfo createFileStorageConfig( + String storageName, String location) { + return FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .setAllowedLocations(List.of(location)) + .setStorageName(storageName) + .build(); + } + + private static PolarisCatalog createS3Catalog( + String catalogName, String storageName, String roleName) { + return PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(new CatalogProperties(s3BaseLocation())) + .setStorageConfigInfo(createS3StorageConfig(storageName, roleName)) + .build(); + } + + private static String s3BaseLocation() { + return System.getProperty(S3_BASE_LOCATION_PROPERTY, DEFAULT_S3_BASE_LOCATION); + } + + private static void requireS3DataPlane() { + Assumptions.assumeTrue( + Boolean.parseBoolean(System.getProperty(S3_DATA_PLANE_ENABLED_PROPERTY, "false")), + "S3 data-plane tests require MinIO/AWS configuration"); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java index e6e3276e54..f2394ce7cb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/NamespaceEntity.java @@ -24,6 +24,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.rest.RESTUtil; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; /** * Namespace-specific subclass of the {@link PolarisEntity} that provides accessors interacting with @@ -74,6 +75,22 @@ public String getBaseLocation() { return getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION); } + /** + * Get the storage configuration for this namespace entity, if present. This allows + * namespace-level storage configuration overrides. + * + * @return the storage configuration, or null if not set + */ + @JsonIgnore + public @Nullable PolarisStorageConfigurationInfo getStorageConfigurationInfo() { + String configStr = + getInternalPropertiesAsMap().get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + if (configStr != null) { + return PolarisStorageConfigurationInfo.deserialize(configStr); + } + return null; + } + public static class Builder extends PolarisEntity.BaseBuilder { public Builder(Namespace namespace) { super(); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/TableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/TableLikeEntity.java index cb840824a9..6c09071746 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/TableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/TableLikeEntity.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Preconditions; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTUtil; @@ -28,7 +29,9 @@ import org.apache.polaris.core.entity.NamespaceEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; /** * An entity type for all table-like entities including Iceberg tables, Iceberg views, and generic @@ -57,4 +60,20 @@ public Namespace getParentNamespace() { } return RESTUtil.decodeNamespace(encodedNamespace); } + + /** + * Get the storage configuration for this table entity, if present. This allows table-level + * storage configuration overrides. + * + * @return the storage configuration, or null if not set + */ + @JsonIgnore + public @Nullable PolarisStorageConfigurationInfo getStorageConfigurationInfo() { + String configStr = + getInternalPropertiesAsMap().get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + if (configStr != null) { + return PolarisStorageConfigurationInfo.deserialize(configStr); + } + return null; + } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/entity/NamespaceEntityStorageConfigTest.java b/polaris-core/src/test/java/org/apache/polaris/core/entity/NamespaceEntityStorageConfigTest.java new file mode 100644 index 0000000000..5b9a351f92 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/entity/NamespaceEntityStorageConfigTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.entity; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; +import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for storage configuration support in NamespaceEntity. + * + *

Tests storage config getter method for namespace entities. + */ +public class NamespaceEntityStorageConfigTest { + + /** Test serialization round-trip for Namespace entity with storage config. */ + @Test + public void testNamespaceEntityStorageConfigRoundTrip() { + // Create a storage config + PolarisStorageConfigurationInfo originalConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://my-bucket/warehouse/namespace1") + .roleARN("arn:aws:iam::123456789012:role/namespace-role") + .region("us-east-1") + .build(); + + // Create a Namespace entity using the Builder with internal properties + NamespaceEntity entity = + new NamespaceEntity.Builder(Namespace.of("namespace1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .setBaseLocation("s3://my-bucket/warehouse/namespace1") + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + originalConfig.serialize()) + .build(); + + // Verify serialization happened (internal property should have JSON) + String storedJson = + entity + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + assertThat(storedJson).isNotNull(); + assertThat(storedJson).isNotEmpty(); + + // Verify retrieval and deserialization + PolarisStorageConfigurationInfo retrievedConfig = entity.getStorageConfigurationInfo(); + assertThat(retrievedConfig).isNotNull(); + assertThat(retrievedConfig.serialize()).isEqualTo(originalConfig.serialize()); + } + + /** Test nested namespace with storage config. */ + @Test + public void testNestedNamespaceStorageConfig() { + // Create a storage config + PolarisStorageConfigurationInfo originalConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://my-bucket/warehouse/ns1/ns2") + .roleARN("arn:aws:iam::123456789012:role/nested-namespace-role") + .region("us-west-2") + .build(); + + // Create a nested Namespace entity + NamespaceEntity entity = + new NamespaceEntity.Builder(Namespace.of("ns1", "ns2")) + .setCatalogId(1L) + .setId(20L) + .setParentId(10L) + .setBaseLocation("s3://my-bucket/warehouse/ns1/ns2") + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + originalConfig.serialize()) + .build(); + + // Verify retrieval + PolarisStorageConfigurationInfo retrievedConfig = entity.getStorageConfigurationInfo(); + assertThat(retrievedConfig).isNotNull(); + assertThat(retrievedConfig.serialize()).isEqualTo(originalConfig.serialize()); + + // Verify namespace hierarchy + assertThat(entity.asNamespace()).isEqualTo(Namespace.of("ns1", "ns2")); + } + + /** Test entity with no storage config returns null. */ + @Test + public void testNamespaceEntityWithNoConfigReturnsNull() { + // Create an entity without storage config + NamespaceEntity entity = + new NamespaceEntity.Builder(Namespace.of("test_namespace")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .build(); + + // Verify getStorageConfigurationInfo() returns null + assertThat(entity.getStorageConfigurationInfo()).isNull(); + } + + /** Test Azure storage config on namespace. */ + @Test + public void testNamespaceWithAzureStorageConfig() { + // Create Azure storage config + PolarisStorageConfigurationInfo config = + AzureStorageConfigurationInfo.builder() + .addAllowedLocations("abfss://container@myaccount.dfs.core.windows.net/namespace1") + .tenantId("test-tenant-id") + .build(); + + // Create namespace with Azure config + NamespaceEntity entity = + new NamespaceEntity.Builder(Namespace.of("azure_namespace")) + .setCatalogId(1L) + .setId(30L) + .setParentId(1L) + .setBaseLocation("abfss://container@myaccount.dfs.core.windows.net/namespace1") + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), config.serialize()) + .build(); + + // Verify retrieval + PolarisStorageConfigurationInfo retrievedConfig = entity.getStorageConfigurationInfo(); + assertThat(retrievedConfig).isNotNull(); + assertThat(retrievedConfig).isInstanceOf(AzureStorageConfigurationInfo.class); + assertThat(retrievedConfig.serialize()).isEqualTo(config.serialize()); + } + + /** Test that base location and storage config work together. */ + @Test + public void testNamespaceWithBaseLocationAndStorageConfig() { + String baseLocation = "s3://my-bucket/warehouse/namespace_with_config"; + + PolarisStorageConfigurationInfo config = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation(baseLocation) + .roleARN("arn:aws:iam::123456789012:role/namespace-role") + .region("eu-central-1") + .build(); + + NamespaceEntity entity = + new NamespaceEntity.Builder(Namespace.of("namespace_with_config")) + .setCatalogId(1L) + .setId(40L) + .setParentId(1L) + .setBaseLocation(baseLocation) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), config.serialize()) + .build(); + + // Verify both base location and storage config are available + assertThat(entity.getBaseLocation()).isEqualTo(baseLocation); + assertThat(entity.getStorageConfigurationInfo()).isNotNull(); + assertThat(entity.getStorageConfigurationInfo().getAllowedLocations()).contains(baseLocation); + } + + /** Test empty namespace (root level). */ + @Test + public void testRootLevelNamespaceWithStorageConfig() { + PolarisStorageConfigurationInfo config = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://my-bucket/warehouse/root") + .roleARN("arn:aws:iam::123456789012:role/root-ns-role") + .region("us-west-1") + .build(); + + NamespaceEntity entity = + new NamespaceEntity.Builder(Namespace.of("root")) + .setCatalogId(1L) + .setId(50L) + .setParentId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), config.serialize()) + .build(); + + assertThat(entity.getStorageConfigurationInfo()).isNotNull(); + assertThat(entity.getParentNamespace()).isEqualTo(Namespace.empty()); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/entity/table/TableLikeEntityStorageConfigTest.java b/polaris-core/src/test/java/org/apache/polaris/core/entity/table/TableLikeEntityStorageConfigTest.java new file mode 100644 index 0000000000..ecad9888ac --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/entity/table/TableLikeEntityStorageConfigTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.entity.table; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for storage configuration support in TableLikeEntity. + * + *

Tests storage config getter method for table entities. Since TableLikeEntity is abstract, we + * use IcebergTableLikeEntity as the concrete implementation. + */ +public class TableLikeEntityStorageConfigTest { + + /** Test serialization round-trip for IcebergTable entity with storage config. */ + @Test + public void testIcebergTableEntityStorageConfigRoundTrip() { + // Create a storage config + PolarisStorageConfigurationInfo originalConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://my-bucket/warehouse/table1") + .roleARN("arn:aws:iam::123456789012:role/example-role") + .region("us-west-2") + .build(); + + // Create an IcebergTable entity using the Builder with internal properties + IcebergTableLikeEntity entity = + new IcebergTableLikeEntity.Builder( + org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "test_table"), + "s3://my-bucket/warehouse/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + originalConfig.serialize()) + .build(); + + // Verify serialization happened (internal property should have JSON) + String storedJson = + entity + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + assertThat(storedJson).isNotNull(); + assertThat(storedJson).isNotEmpty(); + + // Verify retrieval and deserialization + PolarisStorageConfigurationInfo retrievedConfig = entity.getStorageConfigurationInfo(); + assertThat(retrievedConfig).isNotNull(); + assertThat(retrievedConfig.serialize()).isEqualTo(originalConfig.serialize()); + } + + /** Test GenericTable entity with storage config. */ + @Test + public void testGenericTableEntityStorageConfigRoundTrip() { + // Create a storage config + PolarisStorageConfigurationInfo originalConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://my-bucket/warehouse/generic_table") + .roleARN("arn:aws:iam::123456789012:role/generic-table-role") + .region("eu-west-1") + .build(); + + // Create a GenericTable entity + GenericTableEntity entity = + new GenericTableEntity.Builder(TableIdentifier.of("namespace1", "generic_test"), "parquet") + .setCatalogId(1L) + .setId(200L) + .setParentId(10L) + .setBaseLocation("s3://my-bucket/warehouse/generic_table") + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + originalConfig.serialize()) + .build(); + + // Verify serialization happened + String storedJson = + entity + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + assertThat(storedJson).isNotNull(); + + // Verify retrieval and deserialization + PolarisStorageConfigurationInfo retrievedConfig = entity.getStorageConfigurationInfo(); + assertThat(retrievedConfig).isNotNull(); + assertThat(retrievedConfig.serialize()).isEqualTo(originalConfig.serialize()); + } + + /** Test entity with no storage config returns null. */ + @Test + public void testTableEntityWithNoConfigReturnsNull() { + // Create an entity without storage config + IcebergTableLikeEntity entity = + new IcebergTableLikeEntity.Builder( + org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "test_table"), + "s3://my-bucket/warehouse/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .build(); + + // Verify getStorageConfigurationInfo() returns null + assertThat(entity.getStorageConfigurationInfo()).isNull(); + } + + /** + * Test that StorageConfigurationInfo can be retrieved after setting through internal properties. + */ + @Test + public void testStorageConfigThroughInternalProperties() { + PolarisStorageConfigurationInfo config = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://my-bucket/warehouse/builder-test") + .roleARN("arn:aws:iam::123456789012:role/builder-test-role") + .region("ap-south-1") + .build(); + + // Create entity with storage config via builder internal properties + IcebergTableLikeEntity entity = + new IcebergTableLikeEntity.Builder( + org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "builder_test_table"), + "s3://my-bucket/warehouse/builder_test_table/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), config.serialize()) + .build(); + + // Verify retrieval + PolarisStorageConfigurationInfo retrievedConfig = entity.getStorageConfigurationInfo(); + assertThat(retrievedConfig).isNotNull(); + assertThat(retrievedConfig.serialize()).isEqualTo(config.serialize()); + } + + /** Test IcebergView entity can also have storage config. */ + @Test + public void testIcebergViewEntityStorageConfig() { + PolarisStorageConfigurationInfo config = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://my-bucket/warehouse/view1") + .roleARN("arn:aws:iam::123456789012:role/view-role") + .region("us-east-1") + .build(); + + // Create an IcebergView entity + IcebergTableLikeEntity entity = + new IcebergTableLikeEntity.Builder( + org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_VIEW, + TableIdentifier.of("namespace1", "test_view"), + "s3://my-bucket/warehouse/view1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(300L) + .setParentId(10L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), config.serialize()) + .build(); + + // Verify retrieval + PolarisStorageConfigurationInfo retrievedConfig = entity.getStorageConfigurationInfo(); + assertThat(retrievedConfig).isNotNull(); + assertThat(retrievedConfig.serialize()).isEqualTo(config.serialize()); + } +} diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/StorageConfigIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/StorageConfigIT.java new file mode 100644 index 0000000000..314cc705b1 --- /dev/null +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/StorageConfigIT.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.net.URI; +import java.util.Map; +import org.apache.polaris.service.it.test.PolarisStorageConfigIntegrationTest; +import org.apache.polaris.test.minio.Minio; +import org.apache.polaris.test.minio.MinioAccess; +import org.apache.polaris.test.minio.MinioExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; + +@QuarkusIntegrationTest +@TestProfile(StorageConfigIT.Profile.class) +@ExtendWith(MinioExtension.class) +public class StorageConfigIT extends PolarisStorageConfigIntegrationTest { + private static final String MINIO_ACCESS_KEY = "storage-config-ak"; + private static final String MINIO_SECRET_KEY = "storage-config-sk"; + private static final String S3_BASE_LOCATION_PROPERTY = "polaris.it.storage.s3.base-location"; + private static final String S3_ENDPOINT_PROPERTY = "polaris.it.storage.s3.endpoint"; + private static final String S3_PATH_STYLE_ACCESS_PROPERTY = + "polaris.it.storage.s3.path-style-access"; + private static final String S3_DATA_PLANE_ENABLED_PROPERTY = "polaris.it.storage.s3.enabled"; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) + .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") + .build(); + } + } + + @BeforeAll + static void setupMinio( + @Minio(accessKey = MINIO_ACCESS_KEY, secretKey = MINIO_SECRET_KEY) MinioAccess minioAccess) { + URI storageBase = minioAccess.s3BucketUri("/storage-config-it"); + System.setProperty(S3_BASE_LOCATION_PROPERTY, storageBase.toString()); + System.setProperty(S3_ENDPOINT_PROPERTY, minioAccess.s3endpoint()); + System.setProperty(S3_PATH_STYLE_ACCESS_PROPERTY, "true"); + System.setProperty(S3_DATA_PLANE_ENABLED_PROPERTY, "true"); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index b47f4790a8..ee28ff2af1 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -188,6 +188,117 @@ private PolarisResolutionManifest newResolutionManifest(@Nullable String catalog return resolutionManifestFactory.createResolutionManifest(polarisPrincipal, catalogName); } + /** Public helper to resolve a namespace entity. Used by storage config management endpoints. */ + public PolarisEntity resolveNamespaceEntity(String catalogName, String namespaceParam) { + return resolveNamespacePath(catalogName, namespaceParam).getRawLeafEntity(); + } + + /** + * Resolves the full hierarchical path for a namespace, including all ancestor namespaces and the + * catalog root. The returned wrapper can be passed directly to {@link + * org.apache.polaris.service.catalog.io.FileIOUtil#findStorageInfoFromHierarchy} to perform a + * leaf-to-root storage config walk. + * + * @param catalogName the name of the catalog that owns the namespace + * @param namespaceParam the namespace, encoded as a unit-separator-delimited ({@code \u001F}) + * string, e.g. {@code "ns1\u001Fns2\u001Fns3"} + * @return the resolved path wrapper containing every entity from catalog root to leaf namespace + * @throws org.apache.iceberg.exceptions.NotFoundException if the namespace does not exist + */ + public PolarisResolvedPathWrapper resolveNamespacePath( + String catalogName, String namespaceParam) { + // Parse namespace parameter — it is a single string with unit separator (0x1F) between parts + List namespaceParts = List.of(namespaceParam.split("\u001F")); + PolarisResolutionManifest manifest = newResolutionManifest(catalogName); + String key = "namespace"; + manifest.addPath(new ResolverPath(namespaceParts, PolarisEntityType.NAMESPACE), key); + ResolverStatus status = manifest.resolveAll(); + + if (status.getStatus() != ResolverStatus.StatusEnum.SUCCESS) { + throw new NotFoundException( + "Namespace %s not found in catalog %s", namespaceParam, catalogName); + } + + PolarisResolvedPathWrapper resolved = manifest.getResolvedPath(key, true); + if (resolved == null || resolved.getRawLeafEntity() == null) { + throw new NotFoundException( + "Namespace %s not found in catalog %s", namespaceParam, catalogName); + } + + return resolved; + } + + /** Public helper to resolve a table entity. Used by storage config management endpoints. */ + public PolarisEntity resolveTableEntity( + String catalogName, String namespaceParam, String tableName) { + PolarisResolvedPathWrapper resolved = resolveTablePath(catalogName, namespaceParam, tableName); + if (resolved == null || resolved.getRawLeafEntity() == null) { + throw new NotFoundException( + "Table %s.%s not found in catalog %s", namespaceParam, tableName, catalogName); + } + + return resolved.getRawLeafEntity(); + } + + /** + * Public helper to resolve a full table path. Used by storage config management endpoints for + * effective table-level fallback across full namespace ancestry. + */ + public PolarisResolvedPathWrapper resolveTablePath( + String catalogName, String namespaceParam, String tableName) { + List namespaceParts = List.of(namespaceParam.split("\u001F")); + List fullPath = new java.util.ArrayList<>(namespaceParts); + fullPath.add(tableName); + + PolarisResolutionManifest manifest = newResolutionManifest(catalogName); + String key = "table"; + manifest.addPath(new ResolverPath(fullPath, PolarisEntityType.TABLE_LIKE), key); + ResolverStatus status = manifest.resolveAll(); + + if (status.getStatus() != ResolverStatus.StatusEnum.SUCCESS) { + throw new NotFoundException( + "Table %s.%s not found in catalog %s", namespaceParam, tableName, catalogName); + } + + return manifest.getResolvedPath(key, true); + } + + public void updateEntity(long catalogId, PolarisEntity entity) { + // Get the catalog entity to build the catalog path + EntityResult catalogResult = + metaStoreManager.loadEntity( + getCurrentPolarisContext(), 0, catalogId, PolarisEntityType.CATALOG); + + if (!catalogResult.isSuccess()) { + throw new IllegalStateException("Failed to load catalog: " + catalogResult.getReturnStatus()); + } + + CatalogEntity catalogEntity = CatalogEntity.of(catalogResult.getEntity()); + List catalogPath = PolarisEntity.toCoreList(List.of(catalogEntity)); + + EntityResult result = + metaStoreManager.updateEntityPropertiesIfNotChanged( + getCurrentPolarisContext(), catalogPath, entity); + + if (result.isSuccess()) { + return; + } + + BaseResult.ReturnStatus returnStatus = result.getReturnStatus(); + if (returnStatus == BaseResult.ReturnStatus.TARGET_ENTITY_CONCURRENTLY_MODIFIED + || returnStatus == BaseResult.ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED + || returnStatus == BaseResult.ReturnStatus.ENTITY_CANNOT_BE_RESOLVED) { + throw new CommitConflictException( + "Concurrent modification while updating entity '%s'; retry later", entity.getName()); + } + + if (returnStatus == BaseResult.ReturnStatus.ENTITY_NOT_FOUND) { + throw new NotFoundException("Entity %s not found while updating", entity.getName()); + } + + throw new IllegalStateException("Failed to update entity: " + returnStatus); + } + private static PrincipalEntity getPrincipalByName( PolarisResolutionManifest resolutionManifest, String principalName) { return Optional.ofNullable( diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisCatalogsEventServiceDelegator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisCatalogsEventServiceDelegator.java index c316a3d535..b061222965 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisCatalogsEventServiceDelegator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisCatalogsEventServiceDelegator.java @@ -35,6 +35,7 @@ import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.PolicyGrant; import org.apache.polaris.core.admin.model.RevokeGrantRequest; +import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.TableGrant; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; @@ -384,6 +385,71 @@ public Response listGrantsForCatalogRole( return resp; } + @Override + public Response getNamespaceStorageConfig( + String catalogName, + String namespace, + RealmContext realmContext, + SecurityContext securityContext) { + return delegate.getNamespaceStorageConfig( + catalogName, namespace, realmContext, securityContext); + } + + @Override + public Response setNamespaceStorageConfig( + String catalogName, + String namespace, + StorageConfigInfo storageConfigInfo, + RealmContext realmContext, + SecurityContext securityContext) { + return delegate.setNamespaceStorageConfig( + catalogName, namespace, storageConfigInfo, realmContext, securityContext); + } + + @Override + public Response deleteNamespaceStorageConfig( + String catalogName, + String namespace, + RealmContext realmContext, + SecurityContext securityContext) { + return delegate.deleteNamespaceStorageConfig( + catalogName, namespace, realmContext, securityContext); + } + + @Override + public Response getTableStorageConfig( + String catalogName, + String namespace, + String table, + RealmContext realmContext, + SecurityContext securityContext) { + return delegate.getTableStorageConfig( + catalogName, namespace, table, realmContext, securityContext); + } + + @Override + public Response setTableStorageConfig( + String catalogName, + String namespace, + String table, + StorageConfigInfo storageConfigInfo, + RealmContext realmContext, + SecurityContext securityContext) { + return delegate.setTableStorageConfig( + catalogName, namespace, table, storageConfigInfo, realmContext, securityContext); + } + + @Override + public Response deleteTableStorageConfig( + String catalogName, + String namespace, + String table, + RealmContext realmContext, + SecurityContext securityContext) { + return delegate.deleteTableStorageConfig( + catalogName, namespace, table, realmContext, securityContext); + } + private PolarisPrivilege getPrivilegeFromGrantResource(GrantResource grantResource) { return switch (grantResource) { case ViewGrant viewGrant -> PolarisPrivilege.valueOf(viewGrant.getPrivilege().toString()); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index ce94470b62..2710fc04b4 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -23,15 +23,18 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.rest.responses.ErrorResponse; import org.apache.polaris.core.admin.model.AddGrantRequest; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogRole; @@ -43,11 +46,14 @@ import org.apache.polaris.core.admin.model.CreatePrincipalRequest; import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; import org.apache.polaris.core.admin.model.ExternalCatalog; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.GrantResources; import org.apache.polaris.core.admin.model.NamespaceGrant; +import org.apache.polaris.core.admin.model.NamespaceStorageConfigResponse; import org.apache.polaris.core.admin.model.PolicyGrant; import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; @@ -58,6 +64,7 @@ import org.apache.polaris.core.admin.model.RevokeGrantRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.TableGrant; +import org.apache.polaris.core.admin.model.TableStorageConfigResponse; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; @@ -68,15 +75,28 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.CatalogRoleEntity; +import org.apache.polaris.core.entity.NamespaceEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.exceptions.CommitConflictException; import org.apache.polaris.core.identity.provider.ServiceIdentityProvider; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.PrivilegeResult; +import org.apache.polaris.core.storage.FileStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; +import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; +import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; import org.apache.polaris.service.admin.api.PolarisCatalogsApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; +import org.apache.polaris.service.catalog.io.FileIOUtil; import org.apache.polaris.service.config.ReservedProperties; import org.apache.polaris.service.types.PolicyIdentifier; import org.slf4j.Logger; @@ -89,6 +109,31 @@ public class PolarisServiceImpl PolarisPrincipalsApiService, PolarisPrincipalRolesApiService { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisServiceImpl.class); + + /** + * Response header that identifies which level of the hierarchy ({@code TABLE}, {@code NAMESPACE}, + * or {@code CATALOG}) provided the effective storage configuration returned by the GET + * storage-config endpoints. + */ + private static final String STORAGE_CONFIG_SOURCE_HEADER = "X-Polaris-Storage-Config-Source"; + + /** + * Identifies the hierarchy level at which an effective storage configuration was found during a + * leaf-to-root walk (table → namespace(s) → catalog). + */ + private enum StorageConfigSource { + TABLE, + NAMESPACE, + CATALOG + } + + /** + * Pairs a deserialized {@link PolarisStorageConfigurationInfo} with the hierarchy level from + * which it was resolved, enabling callers to populate the {@link #STORAGE_CONFIG_SOURCE_HEADER}. + */ + private record ResolvedStorageConfig( + PolarisStorageConfigurationInfo config, StorageConfigSource source) {} + private final RealmConfig realmConfig; private final ReservedProperties reservedProperties; private final PolarisAdminService adminService; @@ -775,4 +820,684 @@ public Response listGrantsForCatalogRole( GrantResources grantResources = new GrantResources(grantList); return Response.ok(grantResources).build(); } + + /** Storage Configuration Management Endpoints */ + + /** + * Converts an API-layer {@link StorageConfigInfo} into the internal {@link + * PolarisStorageConfigurationInfo} representation for persistence. + * + *

storageName invariant (PR #3409): {@code storageName} must be copied from the API + * model into the internal model for every provider branch. Dropping it silently disables + * named-credential lookup ({@code RESOLVE_CREDENTIALS_BY_STORAGE_NAME}) at the namespace/table + * level. + * + * @param apiModel the API model received from the caller; {@code null} returns {@code null} + * @return the equivalent internal model, ready to be serialized into entity internal properties + * @throws IllegalArgumentException if {@code apiModel} is of an unrecognized subtype + */ + private PolarisStorageConfigurationInfo toInternalModel(StorageConfigInfo apiModel) { + if (apiModel == null) { + return null; + } + + if (apiModel instanceof AwsStorageConfigInfo awsApi) { + return AwsStorageConfigurationInfo.builder() + .addAllAllowedLocations(awsApi.getAllowedLocations()) + .storageName(awsApi.getStorageName()) + .roleARN(awsApi.getRoleArn()) + .externalId(awsApi.getExternalId()) + .userARN(awsApi.getUserArn()) + .currentKmsKey(awsApi.getCurrentKmsKey()) + .allowedKmsKeys(awsApi.getAllowedKmsKeys()) + .region(awsApi.getRegion()) + .endpoint(awsApi.getEndpoint()) + .stsEndpoint(awsApi.getStsEndpoint()) + .endpointInternal(awsApi.getEndpointInternal()) + .pathStyleAccess(awsApi.getPathStyleAccess()) + .stsUnavailable(awsApi.getStsUnavailable()) + .kmsUnavailable(awsApi.getKmsUnavailable()) + .build(); + } + + if (apiModel instanceof AzureStorageConfigInfo azureApi) { + return AzureStorageConfigurationInfo.builder() + .addAllAllowedLocations(azureApi.getAllowedLocations()) + .storageName(azureApi.getStorageName()) + .tenantId(azureApi.getTenantId()) + .multiTenantAppName(azureApi.getMultiTenantAppName()) + .consentUrl(azureApi.getConsentUrl()) + .hierarchical(azureApi.getHierarchical()) + .build(); + } + + if (apiModel instanceof GcpStorageConfigInfo gcpApi) { + return GcpStorageConfigurationInfo.builder() + .addAllAllowedLocations(gcpApi.getAllowedLocations()) + .storageName(gcpApi.getStorageName()) + .gcpServiceAccount(gcpApi.getGcsServiceAccount()) + .build(); + } + + if (apiModel instanceof FileStorageConfigInfo fileApi) { + return FileStorageConfigurationInfo.builder() + .addAllAllowedLocations(fileApi.getAllowedLocations()) + .storageName(fileApi.getStorageName()) + .build(); + } + + throw new IllegalArgumentException("Unsupported storage config type: " + apiModel.getClass()); + } + + /** + * Converts an internal {@link PolarisStorageConfigurationInfo} back into the API-layer {@link + * StorageConfigInfo} for serialization into HTTP responses. + * + *

storageName invariant (PR #3409): {@code storageName} must be copied into every + * provider branch so that GET responses round-trip the value that was PUT. + * + * @param internal the deserialized internal model; {@code null} returns {@code null} + * @return the equivalent API model + * @throws IllegalArgumentException if {@code internal} is of an unrecognized subtype + */ + private StorageConfigInfo toApiModel(PolarisStorageConfigurationInfo internal) { + if (internal == null) { + return null; + } + + if (internal instanceof AwsStorageConfigurationInfo awsInternal) { + return AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setStorageName(awsInternal.getStorageName()) + .setAllowedLocations(awsInternal.getAllowedLocations()) + .setRoleArn(awsInternal.getRoleARN()) + .setExternalId(awsInternal.getExternalId()) + .setUserArn(awsInternal.getUserARN()) + .setCurrentKmsKey(awsInternal.getCurrentKmsKey()) + .setAllowedKmsKeys(awsInternal.getAllowedKmsKeys()) + .setRegion(awsInternal.getRegion()) + .setEndpoint(awsInternal.getEndpoint()) + .setStsEndpoint(awsInternal.getStsEndpoint()) + .setEndpointInternal(awsInternal.getEndpointInternal()) + .setPathStyleAccess(awsInternal.getPathStyleAccess()) + .setStsUnavailable(awsInternal.getStsUnavailable()) + .setKmsUnavailable(awsInternal.getKmsUnavailable()) + .build(); + } + + if (internal instanceof AzureStorageConfigurationInfo azureInternal) { + return AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setStorageName(azureInternal.getStorageName()) + .setAllowedLocations(azureInternal.getAllowedLocations()) + .setTenantId(azureInternal.getTenantId()) + .setMultiTenantAppName(azureInternal.getMultiTenantAppName()) + .setConsentUrl(azureInternal.getConsentUrl()) + .setHierarchical(azureInternal.isHierarchical()) + .build(); + } + + if (internal instanceof GcpStorageConfigurationInfo gcpInternal) { + return GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setStorageName(gcpInternal.getStorageName()) + .setAllowedLocations(gcpInternal.getAllowedLocations()) + .setGcsServiceAccount(gcpInternal.getGcpServiceAccount()) + .build(); + } + + if (internal instanceof FileStorageConfigurationInfo fileInternal) { + return new FileStorageConfigInfo( + StorageConfigInfo.StorageTypeEnum.FILE, + fileInternal.getAllowedLocations(), + fileInternal.getStorageName()); + } + + throw new IllegalArgumentException("Unsupported storage config type: " + internal.getClass()); + } + + // ---- private helpers ----------------------------------------------------------------------- + + private CatalogEntity getCatalogEntity(String catalogName) { + return adminService.getCatalog(catalogName); + } + + private PolarisEntity resolveNamespaceEntity(String catalogName, String namespaceStr) { + return adminService.resolveNamespaceEntity(catalogName, namespaceStr); + } + + private PolarisEntity resolveTableEntity( + String catalogName, String namespaceStr, String tableName) { + return adminService.resolveTableEntity(catalogName, namespaceStr, tableName); + } + + /** + * Resolves the effective storage configuration for a namespace GET request. + * + *

Walks the full entity path from the target namespace up through every ancestor namespace to + * the catalog root, returning the configuration found at the most-specific level. This mirrors + * the leaf-to-root walk performed for table-level resolution and ensures that a deeply nested + * namespace (e.g. {@code ns1\u001Fns2\u001Fns3}) correctly inherits a config set on an ancestor + * namespace rather than falling through directly to the catalog. + * + *

Resolution order: namespace → parent namespace(s) → catalog. + * + * @param catalogName the catalog that owns the namespace + * @param namespace the unit-separator-delimited namespace string (e.g. {@code "ns1\u001Fns2"}) + * @return the resolved config together with its source level, or {@code null} if no storage + * config is present anywhere in the hierarchy + */ + private ResolvedStorageConfig resolveEffectiveNamespaceStorageConfig( + String catalogName, String namespace) { + PolarisResolvedPathWrapper resolvedPath = + adminService.resolveNamespacePath(catalogName, namespace); + + PolarisEntity resolvedEntity = + FileIOUtil.findStorageInfoFromHierarchy(resolvedPath).orElse(null); + if (resolvedEntity == null) { + return null; + } + + PolarisStorageConfigurationInfo storageConfig = storageConfigFromEntity(resolvedEntity); + if (storageConfig == null) { + return null; + } + + return new ResolvedStorageConfig(storageConfig, sourceForEntityType(resolvedEntity.getType())); + } + + /** + * Resolves the effective storage configuration for a table GET request. + * + *

Delegates to {@link org.apache.polaris.service.admin.PolarisAdminService#resolveTablePath} + * to build the complete entity path (catalog → namespace ancestry → table) and then calls {@link + * org.apache.polaris.service.catalog.io.FileIOUtil#findStorageInfoFromHierarchy} to walk that + * path from the table leaf back to the catalog root, stopping at the first entity with an inline + * storage configuration. + * + *

Resolution order: table → namespace ancestry → catalog. + * + * @param catalogName the catalog that owns the table + * @param namespace the unit-separator-delimited namespace string + * @param table the table name + * @return the resolved config together with its source level, or {@code null} if no storage + * config is present anywhere in the hierarchy + */ + private ResolvedStorageConfig resolveEffectiveTableStorageConfigWithSource( + String catalogName, String namespace, String table) { + // resolveTablePath validates that both the namespace and table exist, throwing + // NotFoundException if either is absent — no need for separate pre-checks. + PolarisResolvedPathWrapper resolvedPath = + adminService.resolveTablePath(catalogName, namespace, table); + + PolarisEntity resolvedEntity = + FileIOUtil.findStorageInfoFromHierarchy(resolvedPath).orElse(null); + if (resolvedEntity == null) { + return null; + } + + PolarisStorageConfigurationInfo storageConfig = storageConfigFromEntity(resolvedEntity); + if (storageConfig == null) { + return null; + } + + return new ResolvedStorageConfig(storageConfig, sourceForEntityType(resolvedEntity.getType())); + } + + /** + * Maps a {@link PolarisEntityType} to the corresponding {@link StorageConfigSource} enum constant + * used in the {@link #STORAGE_CONFIG_SOURCE_HEADER} response header. + */ + private StorageConfigSource sourceForEntityType(PolarisEntityType entityType) { + if (entityType == PolarisEntityType.TABLE_LIKE) { + return StorageConfigSource.TABLE; + } + if (entityType == PolarisEntityType.NAMESPACE) { + return StorageConfigSource.NAMESPACE; + } + if (entityType == PolarisEntityType.CATALOG) { + return StorageConfigSource.CATALOG; + } + throw new IllegalArgumentException( + "Unsupported entity type for storage config source: " + entityType); + } + + /** + * Deserializes the storage configuration stored in an entity's internal properties, dispatching + * to the appropriate typed accessor based on the entity type. + * + * @param entity a TABLE_LIKE, NAMESPACE, or CATALOG entity that may carry inline storage config + * @return the deserialized config, or {@code null} if the entity type is unrecognized + */ + private PolarisStorageConfigurationInfo storageConfigFromEntity(PolarisEntity entity) { + if (entity.getType() == PolarisEntityType.TABLE_LIKE) { + return IcebergTableLikeEntity.of(entity).getStorageConfigurationInfo(); + } + if (entity.getType() == PolarisEntityType.NAMESPACE) { + return NamespaceEntity.of(entity).getStorageConfigurationInfo(); + } + if (entity.getType() == PolarisEntityType.CATALOG) { + return CatalogEntity.of(entity).getStorageConfigurationInfo(); + } + return null; + } + + @Override + public Response getNamespaceStorageConfig( + String catalogName, + String namespace, + RealmContext realmContext, + SecurityContext securityContext) { + try { + ResolvedStorageConfig resolved = + resolveEffectiveNamespaceStorageConfig(catalogName, namespace); + if (resolved == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage("No storage configuration found for namespace") + .build()) + .build(); + } + + StorageConfigInfo apiModel = toApiModel(resolved.config()); + return Response.ok(apiModel) + .header(STORAGE_CONFIG_SOURCE_HEADER, resolved.source().name()) + .build(); + + } catch (ForbiddenException e) { + return Response.status(Response.Status.FORBIDDEN) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.FORBIDDEN.getStatusCode()) + .withType("FORBIDDEN") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (jakarta.ws.rs.NotFoundException e) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (Exception e) { + LOGGER.error("Error getting namespace storage config", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .withType("INTERNAL_ERROR") + .withMessage("Internal error: " + e.getMessage()) + .build()) + .build(); + } + } + + @Override + public Response setNamespaceStorageConfig( + String catalogName, + String namespace, + StorageConfigInfo storageConfigInfo, + RealmContext realmContext, + SecurityContext securityContext) { + try { + // Validate storage config + validateStorageConfig(storageConfigInfo); + + // Resolve the namespace entity + PolarisEntity namespaceEntity = resolveNamespaceEntity(catalogName, namespace); + CatalogEntity catalogEntity = getCatalogEntity(catalogName); + + // Convert API model to internal model + PolarisStorageConfigurationInfo internalConfig = toInternalModel(storageConfigInfo); + + // Update entity with new config using builder + PolarisEntity.Builder entityBuilder = new PolarisEntity.Builder(namespaceEntity); + entityBuilder.addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), internalConfig.serialize()); + PolarisEntity updatedEntity = entityBuilder.build(); + + // Persist to metastore + adminService.updateEntity(catalogEntity.getId(), updatedEntity); + + // Build proper response with namespace parts + List namespaceParts = List.of(namespace.split("\\u001F")); + NamespaceStorageConfigResponse response = + NamespaceStorageConfigResponse.builder(namespaceParts, storageConfigInfo) + .setMessage("Storage configuration updated successfully") + .build(); + + return Response.ok(response).build(); + + } catch (CommitConflictException e) { + return Response.status(Response.Status.CONFLICT) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.CONFLICT.getStatusCode()) + .withType("CONFLICT") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (ForbiddenException e) { + return Response.status(Response.Status.FORBIDDEN) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.FORBIDDEN.getStatusCode()) + .withType("FORBIDDEN") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (jakarta.ws.rs.NotFoundException e) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (IllegalArgumentException e) { + LOGGER.warn("Invalid storage config", e); + return Response.status(Response.Status.BAD_REQUEST) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.BAD_REQUEST.getStatusCode()) + .withType("BAD_REQUEST") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (Exception e) { + LOGGER.error("Error setting namespace storage config", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .withType("INTERNAL_ERROR") + .withMessage("Internal error: " + e.getMessage()) + .build()) + .build(); + } + } + + @Override + public Response deleteNamespaceStorageConfig( + String catalogName, + String namespace, + RealmContext realmContext, + SecurityContext securityContext) { + try { + // Resolve the namespace entity + PolarisEntity namespaceEntity = resolveNamespaceEntity(catalogName, namespace); + CatalogEntity catalogEntity = getCatalogEntity(catalogName); + + // Remove storage configuration using builder + Map internalProps = + new HashMap<>(namespaceEntity.getInternalPropertiesAsMap()); + internalProps.remove(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + + PolarisEntity updatedEntity = + new PolarisEntity.Builder(namespaceEntity).setInternalProperties(internalProps).build(); + + // Persist to metastore + adminService.updateEntity(catalogEntity.getId(), updatedEntity); + + // Return 204 No Content + return Response.noContent().build(); + + } catch (CommitConflictException e) { + return Response.status(Response.Status.CONFLICT) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.CONFLICT.getStatusCode()) + .withType("CONFLICT") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (ForbiddenException e) { + return Response.status(Response.Status.FORBIDDEN) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.FORBIDDEN.getStatusCode()) + .withType("FORBIDDEN") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (jakarta.ws.rs.NotFoundException e) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (Exception e) { + LOGGER.error("Error deleting namespace storage config", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .withType("INTERNAL_ERROR") + .withMessage("Internal error: " + e.getMessage()) + .build()) + .build(); + } + } + + @Override + public Response getTableStorageConfig( + String catalogName, + String namespace, + String table, + RealmContext realmContext, + SecurityContext securityContext) { + try { + getCatalogEntity(catalogName); + + ResolvedStorageConfig resolved = + resolveEffectiveTableStorageConfigWithSource(catalogName, namespace, table); + + if (resolved == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage("No storage configuration found for table") + .build()) + .build(); + } + + StorageConfigInfo apiModel = toApiModel(resolved.config()); + return Response.ok(apiModel) + .header(STORAGE_CONFIG_SOURCE_HEADER, resolved.source().name()) + .build(); + + } catch (ForbiddenException e) { + return Response.status(Response.Status.FORBIDDEN) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.FORBIDDEN.getStatusCode()) + .withType("FORBIDDEN") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (jakarta.ws.rs.NotFoundException e) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (Exception e) { + LOGGER.error("Error getting table storage config", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .withType("INTERNAL_ERROR") + .withMessage("Internal error: " + e.getMessage()) + .build()) + .build(); + } + } + + @Override + public Response setTableStorageConfig( + String catalogName, + String namespace, + String table, + StorageConfigInfo storageConfigInfo, + RealmContext realmContext, + SecurityContext securityContext) { + try { + // Validate storage config + validateStorageConfig(storageConfigInfo); + + // Resolve the table entity + PolarisEntity tableEntity = resolveTableEntity(catalogName, namespace, table); + CatalogEntity catalogEntity = getCatalogEntity(catalogName); + + // Convert API model to internal model + PolarisStorageConfigurationInfo internalConfig = toInternalModel(storageConfigInfo); + + // Update entity with new config using builder + PolarisEntity.Builder entityBuilder = new PolarisEntity.Builder(tableEntity); + entityBuilder.addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), internalConfig.serialize()); + PolarisEntity updatedEntity = entityBuilder.build(); + + // Persist to metastore + adminService.updateEntity(catalogEntity.getId(), updatedEntity); + + // Build proper response with namespace parts and table name + List namespaceParts = List.of(namespace.split("\\u001F")); + TableStorageConfigResponse response = + TableStorageConfigResponse.builder(namespaceParts, table, storageConfigInfo) + .setMessage("Storage configuration updated successfully") + .build(); + + return Response.ok(response).build(); + + } catch (CommitConflictException e) { + return Response.status(Response.Status.CONFLICT) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.CONFLICT.getStatusCode()) + .withType("CONFLICT") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (ForbiddenException e) { + return Response.status(Response.Status.FORBIDDEN) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.FORBIDDEN.getStatusCode()) + .withType("FORBIDDEN") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (jakarta.ws.rs.NotFoundException e) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (IllegalArgumentException e) { + LOGGER.warn("Invalid storage config", e); + return Response.status(Response.Status.BAD_REQUEST) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.BAD_REQUEST.getStatusCode()) + .withType("BAD_REQUEST") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (Exception e) { + LOGGER.error("Error setting table storage config", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .withType("INTERNAL_ERROR") + .withMessage("Internal error: " + e.getMessage()) + .build()) + .build(); + } + } + + @Override + public Response deleteTableStorageConfig( + String catalogName, + String namespace, + String table, + RealmContext realmContext, + SecurityContext securityContext) { + try { + // Resolve the table entity + PolarisEntity tableEntity = resolveTableEntity(catalogName, namespace, table); + CatalogEntity catalogEntity = getCatalogEntity(catalogName); + + // Remove storage configuration using builder + Map internalProps = new HashMap<>(tableEntity.getInternalPropertiesAsMap()); + internalProps.remove(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + + PolarisEntity updatedEntity = + new PolarisEntity.Builder(tableEntity).setInternalProperties(internalProps).build(); + + // Persist to metastore + adminService.updateEntity(catalogEntity.getId(), updatedEntity); + + // Return 204 No Content + return Response.noContent().build(); + + } catch (CommitConflictException e) { + return Response.status(Response.Status.CONFLICT) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.CONFLICT.getStatusCode()) + .withType("CONFLICT") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (ForbiddenException e) { + return Response.status(Response.Status.FORBIDDEN) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.FORBIDDEN.getStatusCode()) + .withType("FORBIDDEN") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (jakarta.ws.rs.NotFoundException e) { + return Response.status(Response.Status.NOT_FOUND) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.NOT_FOUND.getStatusCode()) + .withType("NOT_FOUND") + .withMessage(e.getMessage()) + .build()) + .build(); + } catch (Exception e) { + LOGGER.error("Error deleting table storage config", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity( + ErrorResponse.builder() + .responseCode(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .withType("INTERNAL_ERROR") + .withMessage("Internal error: " + e.getMessage()) + .build()) + .build(); + } + } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 776a0a8b4a..42ae62e66a 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -2170,6 +2170,26 @@ private void validateMetadataFileInTableDir(TableIdentifier identifier, TableMet } } + /** + * Loads FileIO for table operations with hierarchical storage config resolution. + * + *

The {@code resolvedStorageEntity} contains the complete entity hierarchy (catalog → + * namespace(s) → table). Storage configuration is resolved hierarchically with the following + * priority order: table > namespace(s) > catalog. + * + *

This method delegates to {@link StorageAccessConfigProvider} which uses {@link FileIOUtil} + * to walk the entity hierarchy backward (leaf to root) to find the first entity with storage + * configuration. This enables tables and namespaces to override storage configuration inherited + * from their parent entities. + * + * @param identifier the table identifier for logging and credential vending + * @param readLocations set of storage locations that will be read from + * @param resolvedStorageEntity complete resolved path with all namespace levels in the hierarchy + * @param tableProperties Iceberg table properties for FileIO initialization + * @param storageActions the storage operations to scope credentials to (READ, WRITE, LIST, + * DELETE) + * @return A configured {@link FileIO} instance with appropriately scoped credentials + */ private FileIO loadFileIOForTableLike( TableIdentifier identifier, Set readLocations, diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java index 36977a7e02..428299bba7 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java @@ -22,9 +22,13 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileIOUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(FileIOUtil.class); + private FileIOUtil() {} /** @@ -35,11 +39,18 @@ private FileIOUtil() {} * properties, identified using a key from {@link * PolarisEntityConstants#getStorageConfigInfoPropertyName()}. * + *

This method returns the entity itself rather than the deserialized configuration to support + * caching and other entity-based operations in the credential vending flow. + * + *

Resolution order (backwards): Table → Namespace(s) → Catalog + * * @param resolvedStorageEntity the resolved entity wrapper containing the hierarchical path * @return an {@link Optional} containing the entity with storage config, or empty if not found */ public static Optional findStorageInfoFromHierarchy( PolarisResolvedPathWrapper resolvedStorageEntity) { + // Walk the path in reverse (leaf to root: table → namespace(s) → catalog) + // This supports hierarchical storage config overrides where table > namespace > catalog Optional storageInfoEntity = resolvedStorageEntity.getRawFullPath().reversed().stream() .filter( @@ -47,6 +58,17 @@ public static Optional findStorageInfoFromHierarchy( e.getInternalPropertiesAsMap() .containsKey(PolarisEntityConstants.getStorageConfigInfoPropertyName())) .findFirst(); + + if (storageInfoEntity.isPresent()) { + LOGGER + .atDebug() + .addKeyValue("entityName", storageInfoEntity.get().getName()) + .addKeyValue("entityType", storageInfoEntity.get().getType()) + .log("Found storage configuration in entity hierarchy"); + } else { + LOGGER.atDebug().log("No storage configuration found in entity hierarchy"); + } + return storageInfoEntity; } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java index 8b8acb9e6b..6b220aa355 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java @@ -75,12 +75,17 @@ public StorageAccessConfigProvider( /** * Vends credentials for accessing table storage at explicit locations. * + *

This method performs hierarchical storage configuration resolution by searching the entity + * path (table → namespace(s) → catalog) to find the most specific storage configuration. This + * enables tables and namespaces to override storage configuration inherited from parent entities. + * * @param tableIdentifier the table identifier, used for logging and refresh endpoint construction * @param tableLocations set of storage location URIs to scope credentials to * @param storageActions the storage operations (READ, WRITE, LIST, DELETE) to scope credentials * to * @param refreshCredentialsEndpoint optional endpoint URL for clients to refresh credentials - * @param resolvedPath the entity hierarchy to search for storage configuration + * @param resolvedPath the complete entity hierarchy (catalog → namespace(s) → table) to search + * for storage configuration * @return {@link StorageAccessConfig} with scoped credentials and metadata; empty if no storage * config found */ diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceTest.java index 276315fb15..7c8d9f78b1 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAdminServiceTest.java @@ -44,6 +44,7 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.exceptions.CommitConflictException; import org.apache.polaris.core.identity.provider.ServiceIdentityProvider; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; @@ -603,6 +604,51 @@ void testGrantPrivilegeOnTableLikeToRole_SyntheticEntityCreationFails() throws E "Failed to create or find table entity 'test-table' in federated catalog 'test-catalog'"); } + @Test + void testUpdateEntity_ConcurrentModificationThrowsCommitConflict() { + EntityResult catalogLoadResult = mock(EntityResult.class); + when(catalogLoadResult.isSuccess()).thenReturn(true); + when(catalogLoadResult.getEntity()) + .thenReturn(createEntity("test-catalog", PolarisEntityType.CATALOG, 1L)); + when(metaStoreManager.loadEntity(any(), eq(0L), eq(1L), eq(PolarisEntityType.CATALOG))) + .thenReturn(catalogLoadResult); + + EntityResult updateResult = mock(EntityResult.class); + when(updateResult.isSuccess()).thenReturn(false); + when(updateResult.getReturnStatus()) + .thenReturn(BaseResult.ReturnStatus.TARGET_ENTITY_CONCURRENTLY_MODIFIED); + when(metaStoreManager.updateEntityPropertiesIfNotChanged(any(), any(), any())) + .thenReturn(updateResult); + + PolarisEntity namespaceEntity = createEntity("test-ns", PolarisEntityType.NAMESPACE, 2L); + + assertThatThrownBy(() -> adminService.updateEntity(1L, namespaceEntity)) + .isInstanceOf(CommitConflictException.class) + .hasMessageContaining("Concurrent modification while updating entity"); + } + + @Test + void testUpdateEntity_EntityNotFoundThrowsNotFound() { + EntityResult catalogLoadResult = mock(EntityResult.class); + when(catalogLoadResult.isSuccess()).thenReturn(true); + when(catalogLoadResult.getEntity()) + .thenReturn(createEntity("test-catalog", PolarisEntityType.CATALOG, 1L)); + when(metaStoreManager.loadEntity(any(), eq(0L), eq(1L), eq(PolarisEntityType.CATALOG))) + .thenReturn(catalogLoadResult); + + EntityResult updateResult = mock(EntityResult.class); + when(updateResult.isSuccess()).thenReturn(false); + when(updateResult.getReturnStatus()).thenReturn(BaseResult.ReturnStatus.ENTITY_NOT_FOUND); + when(metaStoreManager.updateEntityPropertiesIfNotChanged(any(), any(), any())) + .thenReturn(updateResult); + + PolarisEntity namespaceEntity = createEntity("test-ns", PolarisEntityType.NAMESPACE, 2L); + + assertThatThrownBy(() -> adminService.updateEntity(1L, namespaceEntity)) + .isInstanceOf(NotFoundException.class) + .hasMessageContaining("Entity test-ns not found while updating"); + } + private PolarisEntity createEntity(String name, PolarisEntityType type) { return new PolarisEntity.Builder() .setName(name) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java index 2fe0e52680..6c6294895e 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java @@ -18,13 +18,20 @@ */ package org.apache.polaris.service.admin; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.when; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; import java.lang.reflect.Method; import java.util.List; +import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; @@ -37,6 +44,11 @@ import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.RealmConfig; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.exceptions.CommitConflictException; import org.apache.polaris.core.identity.provider.ServiceIdentityProvider; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; @@ -78,6 +90,9 @@ void setUp() { when(realmConfig.getConfig( FeatureConfiguration.SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES)) .thenReturn(List.of("OAUTH")); + when(realmConfig.getConfig(FeatureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES)) + .thenReturn(List.of("S3", "AZURE", "GCS", "FILE")); + when(realmConfig.getConfig(FeatureConfiguration.ALLOW_SETTING_S3_ENDPOINTS)).thenReturn(true); adminService = new PolarisAdminService( @@ -237,4 +252,119 @@ private void invokeValidateExternalCatalog(PolarisServiceImpl service, Catalog c } } } + + @Test + void testSetNamespaceStorageConfigReturnsConflictOnConcurrentUpdate() { + PolarisAdminService mockedAdminService = Mockito.mock(PolarisAdminService.class); + PolarisServiceImpl service = + new PolarisServiceImpl( + realmConfig, reservedProperties, mockedAdminService, serviceIdentityProvider); + + CatalogEntity catalogEntity = + new CatalogEntity.Builder().setId(100L).setName("catalog").build(); + PolarisEntity namespaceEntity = + new PolarisEntity.Builder() + .setId(200L) + .setCatalogId(100L) + .setName("ns") + .setType(PolarisEntityType.NAMESPACE) + .build(); + + when(mockedAdminService.resolveNamespaceEntity("catalog", "ns")).thenReturn(namespaceEntity); + when(mockedAdminService.getCatalog("catalog")).thenReturn(catalogEntity); + Mockito.doThrow(new CommitConflictException("conflict")) + .when(mockedAdminService) + .updateEntity(anyLong(), any(PolarisEntity.class)); + + FileStorageConfigInfo storageConfigInfo = + FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .setAllowedLocations(List.of("file:///tmp/test")) + .build(); + + Response response = + service.setNamespaceStorageConfig( + "catalog", + "ns", + storageConfigInfo, + Mockito.mock(RealmContext.class), + Mockito.mock(SecurityContext.class)); + + assertThat(response.getStatus()).isEqualTo(Response.Status.CONFLICT.getStatusCode()); + response.close(); + } + + @Test + void testSetTableStorageConfigReturnsForbiddenWhenUnauthorized() { + PolarisAdminService mockedAdminService = Mockito.mock(PolarisAdminService.class); + PolarisServiceImpl service = + new PolarisServiceImpl( + realmConfig, reservedProperties, mockedAdminService, serviceIdentityProvider); + + when(mockedAdminService.resolveTableEntity("catalog", "ns", "tbl")) + .thenThrow(new ForbiddenException("denied")); + + AwsStorageConfigInfo storageConfigInfo = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://bucket/path")) + .setRoleArn("arn:aws:iam::123456789012:role/test") + .build(); + + Response response = + service.setTableStorageConfig( + "catalog", + "ns", + "tbl", + storageConfigInfo, + Mockito.mock(RealmContext.class), + Mockito.mock(SecurityContext.class)); + + assertThat(response.getStatus()).isEqualTo(Response.Status.FORBIDDEN.getStatusCode()); + response.close(); + } + + @Test + void testGetTableStorageConfigReturnsForbiddenWhenUnauthorized() { + PolarisAdminService mockedAdminService = Mockito.mock(PolarisAdminService.class); + PolarisServiceImpl service = + new PolarisServiceImpl( + realmConfig, reservedProperties, mockedAdminService, serviceIdentityProvider); + + when(mockedAdminService.resolveTablePath("catalog", "ns", "tbl")) + .thenThrow(new ForbiddenException("denied")); + + Response response = + service.getTableStorageConfig( + "catalog", + "ns", + "tbl", + Mockito.mock(RealmContext.class), + Mockito.mock(SecurityContext.class)); + + assertThat(response.getStatus()).isEqualTo(Response.Status.FORBIDDEN.getStatusCode()); + response.close(); + } + + @Test + void testDeleteTableStorageConfigReturnsForbiddenWhenUnauthorized() { + PolarisAdminService mockedAdminService = Mockito.mock(PolarisAdminService.class); + PolarisServiceImpl service = + new PolarisServiceImpl( + realmConfig, reservedProperties, mockedAdminService, serviceIdentityProvider); + + when(mockedAdminService.resolveTableEntity("catalog", "ns", "tbl")) + .thenThrow(new ForbiddenException("denied")); + + Response response = + service.deleteTableStorageConfig( + "catalog", + "ns", + "tbl", + Mockito.mock(RealmContext.class), + Mockito.mock(SecurityContext.class)); + + assertThat(response.getStatus()).isEqualTo(Response.Status.FORBIDDEN.getStatusCode()); + response.close(); + } } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOUtilTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOUtilTest.java new file mode 100644 index 0000000000..f617b3524c --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOUtilTest.java @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.io; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.NamespaceEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link FileIOUtil} focusing on hierarchical storage configuration resolution. + * + *

These tests verify that storage configuration is correctly resolved from entity hierarchies, + * supporting the pattern: Table → Namespace(s) → Catalog + */ +public class FileIOUtilTest { + + /** + * Test that when a table has storage config, it is found and returned. + * + *

Hierarchy: Catalog → Namespace → Table (with config) + */ + @Test + public void testFindStorageInfo_TableHasConfig() { + // Setup: Create entities + PolarisEntity catalog = createCatalogWithConfig("s3://catalog-bucket/"); + PolarisEntity namespace = createNamespaceWithoutConfig("ns1"); + PolarisEntity table = createTableWithConfig("table1", "s3://table-bucket/"); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = buildResolvedPath(catalog, namespace, table); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Table config is found + assertThat(result).isPresent(); + assertThat(result.get().getName()).isEqualTo("table1"); + assertThat(result.get().getType()).isEqualTo(PolarisEntityType.TABLE_LIKE); + } + + /** + * Test that when a table has no config but namespace does, namespace config is found. + * + *

Hierarchy: Catalog → Namespace (with config) → Table (no config) + */ + @Test + public void testFindStorageInfo_NamespaceHasConfig() { + // Setup: Create entities + PolarisEntity catalog = createCatalogWithConfig("s3://catalog-bucket/"); + PolarisEntity namespace = createNamespaceWithConfig("ns1", "s3://namespace-bucket/"); + PolarisEntity table = createTableWithoutConfig("table1"); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = buildResolvedPath(catalog, namespace, table); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Namespace config is found + assertThat(result).isPresent(); + assertThat(result.get().getName()).isEqualTo("ns1"); + assertThat(result.get().getType()).isEqualTo(PolarisEntityType.NAMESPACE); + } + + /** + * Test that when neither table nor namespace have config, catalog config is found. + * + *

Hierarchy: Catalog (with config) → Namespace (no config) → Table (no config) + */ + @Test + public void testFindStorageInfo_CatalogFallback() { + // Setup: Create entities + PolarisEntity catalog = createCatalogWithConfig("s3://catalog-bucket/"); + PolarisEntity namespace = createNamespaceWithoutConfig("ns1"); + PolarisEntity table = createTableWithoutConfig("table1"); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = buildResolvedPath(catalog, namespace, table); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Catalog config is found + assertThat(result).isPresent(); + assertThat(result.get().getName()).isEqualTo("test-catalog"); + assertThat(result.get().getType()).isEqualTo(PolarisEntityType.CATALOG); + } + + /** + * Test that when no entity in hierarchy has config, empty is returned. + * + *

Hierarchy: Catalog (no config) → Namespace (no config) → Table (no config) + */ + @Test + public void testFindStorageInfo_NoConfigInHierarchy() { + // Setup: Create entities without config + PolarisEntity catalog = createCatalogWithoutConfig(); + PolarisEntity namespace = createNamespaceWithoutConfig("ns1"); + PolarisEntity table = createTableWithoutConfig("table1"); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = buildResolvedPath(catalog, namespace, table); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: No config found + assertThat(result).isEmpty(); + } + + /** + * Test hierarchical resolution with nested namespaces (4 levels deep). + * + *

Hierarchy: Catalog → ns1 → ns2 (with config) → ns3 → Table + * + *

This tests that resolution correctly walks through multiple namespace levels to find + * configuration at an intermediate level. + */ + @Test + public void testFindStorageInfo_NestedNamespacesWithIntermediateConfig() { + // Setup: Create nested namespace hierarchy + PolarisEntity catalog = createCatalogWithConfig("s3://catalog-bucket/"); + PolarisEntity ns1 = createNamespaceWithoutConfig("ns1"); + PolarisEntity ns2 = createNamespaceWithConfig("ns2", "s3://ns2-bucket/"); + PolarisEntity ns3 = createNamespaceWithoutConfig("ns3"); + PolarisEntity table = createTableWithoutConfig("table1"); + + // Build resolved path with all namespace levels + PolarisResolvedPathWrapper resolvedPath = + buildResolvedPath(catalog, List.of(ns1, ns2, ns3), table); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: ns2 config is found (not ns1, ns3, or catalog) + assertThat(result).isPresent(); + assertThat(result.get().getName()).isEqualTo("ns2"); + assertThat(result.get().getType()).isEqualTo(PolarisEntityType.NAMESPACE); + } + + /** + * Test that table config takes priority over namespace config when both exist. + * + *

Hierarchy: Catalog → Namespace (with config) → Table (with config) + */ + @Test + public void testFindStorageInfo_TableOverridesNamespace() { + // Setup: Both namespace and table have configs + PolarisEntity catalog = createCatalogWithConfig("s3://catalog-bucket/"); + PolarisEntity namespace = createNamespaceWithConfig("ns1", "s3://namespace-bucket/"); + PolarisEntity table = createTableWithConfig("table1", "s3://table-bucket/"); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = buildResolvedPath(catalog, namespace, table); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Table config takes priority + assertThat(result).isPresent(); + assertThat(result.get().getName()).isEqualTo("table1"); + } + + /** + * Test resolution with only catalog and namespace (no table). + * + *

Hierarchy: Catalog → Namespace (with config) + */ + @Test + public void testFindStorageInfo_NamespaceOnly() { + // Setup: Create catalog and namespace only + PolarisEntity catalog = createCatalogWithConfig("s3://catalog-bucket/"); + PolarisEntity namespace = createNamespaceWithConfig("ns1", "s3://namespace-bucket/"); + + // Build resolved path without table + PolarisResolvedPathWrapper resolvedPath = buildResolvedPath(catalog, namespace); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Namespace config is found + assertThat(result).isPresent(); + assertThat(result.get().getName()).isEqualTo("ns1"); + } + + /** + * Test resolution with only catalog (no namespace or table). + * + *

Hierarchy: Catalog (with config) + */ + @Test + public void testFindStorageInfo_CatalogOnly() { + // Setup: Create catalog only + PolarisEntity catalog = createCatalogWithConfig("s3://catalog-bucket/"); + + // Build resolved path with only catalog + PolarisResolvedPathWrapper resolvedPath = buildResolvedPath(catalog); + + // Action: Find storage info + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Catalog config is found + assertThat(result).isPresent(); + assertThat(result.get().getName()).isEqualTo("test-catalog"); + } + + // ==================== Helper Methods ==================== + + /** + * Creates a catalog entity with storage configuration. + * + * @param baseLocation the S3 base location for the catalog + * @return a CatalogEntity with storage config in internal properties + */ + private PolarisEntity createCatalogWithConfig(String baseLocation) { + AwsStorageConfigurationInfo storageConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation(baseLocation) + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .externalId("catalog-external-id") + .build(); + + CatalogEntity.Builder builder = + new CatalogEntity.Builder().setName("test-catalog").setCatalogId(1L).setId(1L); + + builder.addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), storageConfig.serialize()); + + return builder.build(); + } + + /** Creates a catalog entity without storage configuration. */ + private PolarisEntity createCatalogWithoutConfig() { + return new CatalogEntity.Builder().setName("test-catalog").setCatalogId(1L).setId(1L).build(); + } + + /** + * Creates a namespace entity with storage configuration. + * + * @param name the namespace name + * @param baseLocation the S3 base location for the namespace + * @return a NamespaceEntity with storage config in internal properties + */ + private PolarisEntity createNamespaceWithConfig(String name, String baseLocation) { + AwsStorageConfigurationInfo storageConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation(baseLocation) + .roleARN("arn:aws:iam::123456789012:role/namespace-role") + .externalId("namespace-external-id") + .build(); + + NamespaceEntity.Builder builder = + new NamespaceEntity.Builder(Namespace.of(name)).setCatalogId(1L).setId(2L); + + builder.addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), storageConfig.serialize()); + + return builder.build(); + } + + /** Creates a namespace entity without storage configuration. */ + private PolarisEntity createNamespaceWithoutConfig(String name) { + return new NamespaceEntity.Builder(Namespace.of(name)).setCatalogId(1L).setId(2L).build(); + } + + /** + * Creates a table entity with storage configuration. + * + * @param name the table name + * @param baseLocation the S3 base location for the table + * @return an IcebergTableLikeEntity with storage config in internal properties + */ + private PolarisEntity createTableWithConfig(String name, String baseLocation) { + AwsStorageConfigurationInfo storageConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation(baseLocation) + .roleARN("arn:aws:iam::123456789012:role/table-role") + .externalId("table-external-id") + .build(); + + IcebergTableLikeEntity.Builder builder = + new IcebergTableLikeEntity.Builder( + org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of(Namespace.of("test_ns"), name), + "s3://test-bucket/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(3L); + + builder.addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), storageConfig.serialize()); + + return builder.build(); + } + + /** Creates a table entity without storage configuration. */ + private PolarisEntity createTableWithoutConfig(String name) { + return new IcebergTableLikeEntity.Builder( + org.apache.polaris.core.entity.PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of(Namespace.of("test_ns"), name), + "s3://test-bucket/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(3L) + .build(); + } + + /** + * Builds a resolved path wrapper from entities. + * + * @param catalog the catalog entity + * @param namespace the namespace entity + * @param table the table entity + * @return a PolarisResolvedPathWrapper containing the full path + */ + private PolarisResolvedPathWrapper buildResolvedPath( + PolarisEntity catalog, PolarisEntity namespace, PolarisEntity table) { + List path = new ArrayList<>(); + path.add(toResolvedEntity(catalog)); + path.add(toResolvedEntity(namespace)); + path.add(toResolvedEntity(table)); + return new PolarisResolvedPathWrapper(path); + } + + /** + * Builds a resolved path wrapper from catalog and namespace only. + * + * @param catalog the catalog entity + * @param namespace the namespace entity + * @return a PolarisResolvedPathWrapper containing the path + */ + private PolarisResolvedPathWrapper buildResolvedPath( + PolarisEntity catalog, PolarisEntity namespace) { + List path = new ArrayList<>(); + path.add(toResolvedEntity(catalog)); + path.add(toResolvedEntity(namespace)); + return new PolarisResolvedPathWrapper(path); + } + + /** + * Builds a resolved path wrapper from catalog only. + * + * @param catalog the catalog entity + * @return a PolarisResolvedPathWrapper containing the path + */ + private PolarisResolvedPathWrapper buildResolvedPath(PolarisEntity catalog) { + List path = new ArrayList<>(); + path.add(toResolvedEntity(catalog)); + return new PolarisResolvedPathWrapper(path); + } + + /** + * Builds a resolved path wrapper with nested namespaces. + * + * @param catalog the catalog entity + * @param namespaces list of namespace entities in order (root to leaf) + * @param table the table entity + * @return a PolarisResolvedPathWrapper containing the full path + */ + private PolarisResolvedPathWrapper buildResolvedPath( + PolarisEntity catalog, List namespaces, PolarisEntity table) { + List path = new ArrayList<>(); + path.add(toResolvedEntity(catalog)); + namespaces.forEach(ns -> path.add(toResolvedEntity(ns))); + path.add(toResolvedEntity(table)); + return new PolarisResolvedPathWrapper(path); + } + + /** + * Converts a PolarisEntity to a ResolvedPolarisEntity. + * + * @param entity the entity to convert + * @return a ResolvedPolarisEntity with empty grant records + */ + private ResolvedPolarisEntity toResolvedEntity(PolarisEntity entity) { + return new ResolvedPolarisEntity(entity, List.of(), List.of()); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/HierarchicalStorageConfigResolutionTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/HierarchicalStorageConfigResolutionTest.java new file mode 100644 index 0000000000..43f84ca513 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/HierarchicalStorageConfigResolutionTest.java @@ -0,0 +1,687 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.io; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.NamespaceEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; +import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; +import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for hierarchical storage configuration resolution. + * + *

These tests verify that storage configuration is correctly resolved from the entity hierarchy + * (table → namespace(s) → catalog) through the production code path: StorageAccessConfigProvider → + * FileIOUtil.findStorageInfoFromHierarchy(). + */ +public class HierarchicalStorageConfigResolutionTest { + + /** Helper method to create ResolvedPolarisEntity with empty grants. */ + private ResolvedPolarisEntity resolved(PolarisEntity entity) { + return new ResolvedPolarisEntity(entity, Collections.emptyList(), Collections.emptyList()); + } + + /** Test that FileIOUtil correctly finds storage config at the table level. */ + @Test + public void testTableLevelStorageConfigOverridesCatalog() { + // Setup: Create catalog with AWS config + PolarisStorageConfigurationInfo catalogConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://catalog-bucket/") + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .region("us-east-1") + .build(); + + CatalogEntity catalog = + new CatalogEntity.Builder() + .setName("test_catalog") + .setId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + catalogConfig.serialize()) + .build(); + + // Setup: Create namespace without storage config + NamespaceEntity namespace = + new NamespaceEntity.Builder(Namespace.of("namespace1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .build(); + + // Setup: Create table with Azure config (different from catalog) + PolarisStorageConfigurationInfo tableConfig = + AzureStorageConfigurationInfo.builder() + .addAllowedLocations( + "abfss://container@myaccount.dfs.core.windows.net/namespace1/table1") + .tenantId("test-tenant-id") + .build(); + + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "table1"), + "abfss://container@myaccount.dfs.core.windows.net/namespace1/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), tableConfig.serialize()) + .build(); + + // Build resolved path: catalog → namespace → table + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(namespace)), + resolved(PolarisEntity.of(table)))); + + // Action: Find storage config from hierarchy + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Table config should be used (not catalog config) + assertThat(result).isPresent(); + assertThat(result.get().getId()).isEqualTo(100L); // Table ID + assertThat(result.get().getName()).isEqualTo("table1"); + + // Verify the config is Azure (not AWS) + PolarisStorageConfigurationInfo foundConfig = + PolarisStorageConfigurationInfo.deserialize( + result + .get() + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName())); + assertThat(foundConfig).isInstanceOf(AzureStorageConfigurationInfo.class); + assertThat(foundConfig.serialize()).isEqualTo(tableConfig.serialize()); + } + + /** Test that namespace config is used when table has no config. */ + @Test + public void testNamespaceLevelStorageConfigUsedByTable() { + // Setup: Create catalog with AWS config + PolarisStorageConfigurationInfo catalogConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://catalog-bucket/") + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .region("us-east-1") + .build(); + + CatalogEntity catalog = + new CatalogEntity.Builder() + .setName("test_catalog") + .setId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + catalogConfig.serialize()) + .build(); + + // Setup: Create namespace with GCP config + PolarisStorageConfigurationInfo namespaceConfig = + GcpStorageConfigurationInfo.builder() + .addAllowedLocation("gs://namespace-bucket/namespace1/") + .build(); + + NamespaceEntity namespace = + new NamespaceEntity.Builder(Namespace.of("namespace1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + namespaceConfig.serialize()) + .build(); + + // Setup: Create table without storage config + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "table1"), + "gs://namespace-bucket/namespace1/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .build(); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(namespace)), + resolved(PolarisEntity.of(table)))); + + // Action: Find storage config from hierarchy + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Namespace config should be used + assertThat(result).isPresent(); + assertThat(result.get().getId()).isEqualTo(10L); // Namespace ID + assertThat(result.get().getName()).isEqualTo("namespace1"); + + // Verify the config is GCP (from namespace, not catalog AWS) + PolarisStorageConfigurationInfo foundConfig = + PolarisStorageConfigurationInfo.deserialize( + result + .get() + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName())); + assertThat(foundConfig).isInstanceOf(GcpStorageConfigurationInfo.class); + assertThat(foundConfig.serialize()).isEqualTo(namespaceConfig.serialize()); + } + + /** + * Test nested namespaces with config at intermediate level. + * + *

Hierarchy: catalog → ns1 (no config) → ns2 (has Azure config) → ns3 (no config) → table (no + * config) + * + *

Expected: ns2's Azure config should be used + */ + @Test + public void testNestedNamespacesIntermediateLevelConfig() { + // Setup: Create catalog with AWS config + PolarisStorageConfigurationInfo catalogConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://catalog-bucket/") + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .region("us-east-1") + .build(); + + CatalogEntity catalog = + new CatalogEntity.Builder() + .setName("test_catalog") + .setId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + catalogConfig.serialize()) + .build(); + + // Setup: Create ns1 (no config) + NamespaceEntity ns1 = + new NamespaceEntity.Builder(Namespace.of("ns1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .build(); + + // Setup: Create ns2 with Azure config + PolarisStorageConfigurationInfo ns2Config = + AzureStorageConfigurationInfo.builder() + .addAllowedLocations("abfss://container@myaccount.dfs.core.windows.net/ns1/ns2/") + .tenantId("test-tenant-id") + .build(); + + NamespaceEntity ns2 = + new NamespaceEntity.Builder(Namespace.of("ns1", "ns2")) + .setCatalogId(1L) + .setId(20L) + .setParentId(10L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), ns2Config.serialize()) + .build(); + + // Setup: Create ns3 (no config) + NamespaceEntity ns3 = + new NamespaceEntity.Builder(Namespace.of("ns1", "ns2", "ns3")) + .setCatalogId(1L) + .setId(30L) + .setParentId(20L) + .build(); + + // Setup: Create table (no config) + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of(Namespace.of("ns1", "ns2", "ns3"), "table1"), + "abfss://container@myaccount.dfs.core.windows.net/ns1/ns2/ns3/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(30L) + .build(); + + // Build resolved path: catalog → ns1 → ns2 → ns3 → table + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(ns1)), + resolved(PolarisEntity.of(ns2)), + resolved(PolarisEntity.of(ns3)), + resolved(PolarisEntity.of(table)))); + + // Action: Find storage config from hierarchy + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: ns2 config should be used (intermediate level) + assertThat(result).isPresent(); + assertThat(result.get().getId()).isEqualTo(20L); // ns2 ID + + // Verify the config is Azure from ns2 + PolarisStorageConfigurationInfo foundConfig = + PolarisStorageConfigurationInfo.deserialize( + result + .get() + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName())); + assertThat(foundConfig).isInstanceOf(AzureStorageConfigurationInfo.class); + assertThat(foundConfig.serialize()).isEqualTo(ns2Config.serialize()); + } + + /** Test catalog fallback when no overrides exist. */ + @Test + public void testCatalogFallbackWhenNoOverrides() { + // Setup: Create catalog with AWS config + PolarisStorageConfigurationInfo catalogConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://catalog-bucket/") + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .region("us-east-1") + .build(); + + CatalogEntity catalog = + new CatalogEntity.Builder() + .setName("test_catalog") + .setId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + catalogConfig.serialize()) + .build(); + + // Setup: Create namespace without config + NamespaceEntity namespace = + new NamespaceEntity.Builder(Namespace.of("namespace1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .build(); + + // Setup: Create table without config + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "table1"), + "s3://catalog-bucket/namespace1/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .build(); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(namespace)), + resolved(PolarisEntity.of(table)))); + + // Action: Find storage config from hierarchy + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Catalog config should be used (fallback) + assertThat(result).isPresent(); + assertThat(result.get().getId()).isEqualTo(1L); // Catalog ID + assertThat(result.get().getName()).isEqualTo("test_catalog"); + + // Verify the config is AWS from catalog + PolarisStorageConfigurationInfo foundConfig = + PolarisStorageConfigurationInfo.deserialize( + result + .get() + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName())); + assertThat(foundConfig).isInstanceOf(AwsStorageConfigurationInfo.class); + assertThat(foundConfig.serialize()).isEqualTo(catalogConfig.serialize()); + } + + /** Test that empty path returns empty result. */ + @Test + public void testEmptyPathReturnsEmpty() { + // Create empty resolved path + PolarisResolvedPathWrapper emptyPath = new PolarisResolvedPathWrapper(Collections.emptyList()); + + // Action: Find storage config from hierarchy + Optional result = FileIOUtil.findStorageInfoFromHierarchy(emptyPath); + + // Assert: Should return empty + assertThat(result).isEmpty(); + } + + /** Test path with no storage config anywhere returns empty. */ + @Test + public void testNoStorageConfigAnywhereReturnsEmpty() { + // Setup: Create catalog without storage config + CatalogEntity catalog = new CatalogEntity.Builder().setName("test_catalog").setId(1L).build(); + + // Setup: Create namespace without storage config + NamespaceEntity namespace = + new NamespaceEntity.Builder(Namespace.of("namespace1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .build(); + + // Setup: Create table without storage config + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "table1"), + "s3://catalog-bucket/namespace1/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .build(); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(namespace)), + resolved(PolarisEntity.of(table)))); + + // Action: Find storage config from hierarchy + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Should return empty (no config found anywhere) + assertThat(result).isEmpty(); + } + + /** + * Test that multiple configs at different levels resolve correctly (closest wins). + * + *

Hierarchy: catalog (AWS) → ns1 (GCP) → ns2 (no config) → table (Azure) + * + *

Expected: Table's Azure config wins + */ + @Test + public void testMultipleConfigsClosestWins() { + // Catalog has AWS + PolarisStorageConfigurationInfo catalogConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://catalog-bucket/") + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .region("us-east-1") + .build(); + + CatalogEntity catalog = + new CatalogEntity.Builder() + .setName("test_catalog") + .setId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + catalogConfig.serialize()) + .build(); + + // ns1 has GCP + PolarisStorageConfigurationInfo ns1Config = + GcpStorageConfigurationInfo.builder().addAllowedLocation("gs://ns1-bucket/").build(); + + NamespaceEntity ns1 = + new NamespaceEntity.Builder(Namespace.of("ns1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), ns1Config.serialize()) + .build(); + + // ns2 has no config + NamespaceEntity ns2 = + new NamespaceEntity.Builder(Namespace.of("ns1", "ns2")) + .setCatalogId(1L) + .setId(20L) + .setParentId(10L) + .build(); + + // Table has Azure + PolarisStorageConfigurationInfo tableConfig = + AzureStorageConfigurationInfo.builder() + .addAllowedLocations("abfss://container@myaccount.dfs.core.windows.net/table") + .tenantId("test-tenant-id") + .build(); + + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of(Namespace.of("ns1", "ns2"), "table1"), + "abfss://container@myaccount.dfs.core.windows.net/table/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(20L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), tableConfig.serialize()) + .build(); + + // Build resolved path + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(ns1)), + resolved(PolarisEntity.of(ns2)), + resolved(PolarisEntity.of(table)))); + + // Action: Find storage config from hierarchy + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + // Assert: Table's Azure config should win (closest to leaf) + assertThat(result).isPresent(); + assertThat(result.get().getId()).isEqualTo(100L); // Table ID + + PolarisStorageConfigurationInfo foundConfig = + PolarisStorageConfigurationInfo.deserialize( + result + .get() + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName())); + assertThat(foundConfig).isInstanceOf(AzureStorageConfigurationInfo.class); + } + + /** + * Test that storageName from the namespace entity takes precedence over storageName at the + * catalog when both have storage configs. + * + *

Hierarchy: catalog (AWS, storageName="cat-creds") → namespace (AWS, storageName="ns-creds") + * → table (no config) + * + *

Expected: namespace entity with storageName="ns-creds" is returned; storageName from the + * catalog ("cat-creds") is NOT in the resolved config. + */ + @Test + public void testStorageNameFromNamespaceTakesPrecedenceOverCatalog() { + PolarisStorageConfigurationInfo catalogConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://catalog-bucket/") + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .region("us-east-1") + .storageName("cat-creds") + .build(); + + CatalogEntity catalog = + new CatalogEntity.Builder() + .setName("test_catalog") + .setId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + catalogConfig.serialize()) + .build(); + + PolarisStorageConfigurationInfo nsConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://ns-bucket/") + .roleARN("arn:aws:iam::123456789012:role/ns-role") + .region("us-east-1") + .storageName("ns-creds") + .build(); + + NamespaceEntity namespace = + new NamespaceEntity.Builder(Namespace.of("namespace1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), nsConfig.serialize()) + .build(); + + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "table1"), + "s3://ns-bucket/namespace1/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .build(); + + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(namespace)), + resolved(PolarisEntity.of(table)))); + + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + assertThat(result).as("Hierarchy walk must find a storage config").isPresent(); + assertThat(result.get().getId()) + .as("Namespace entity (id=10) should be the resolved stop-point, not the catalog") + .isEqualTo(10L); + + PolarisStorageConfigurationInfo foundConfig = + PolarisStorageConfigurationInfo.deserialize( + result + .get() + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName())); + + assertThat(foundConfig.getStorageName()) + .as("Resolved config must carry the namespace storageName 'ns-creds'") + .isEqualTo("ns-creds"); + assertThat(foundConfig.getStorageName()) + .as("Catalog storageName 'cat-creds' must NOT be present in the resolved config") + .isNotEqualTo("cat-creds"); + } + + /** + * Test that a namespace with a storage config that has storageName=null still acts as the + * hierarchy-walk stop-point. The walk must NOT fall through to the catalog even though the + * namespace's storageName is null. + * + *

Hierarchy: catalog (AWS, storageName="cat-creds") → namespace (AWS, storageName=null) → + * table (no config) + * + *

Expected: namespace entity is returned; storageName is null; credential provider will use + * the default AWS chain — not the catalog's named "cat-creds" credentials. + */ + @Test + public void testNullStorageNameAtNamespaceStillStopsHierarchyWalk() { + PolarisStorageConfigurationInfo catalogConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://catalog-bucket/") + .roleARN("arn:aws:iam::123456789012:role/catalog-role") + .region("us-east-1") + .storageName("cat-creds") + .build(); + + CatalogEntity catalog = + new CatalogEntity.Builder() + .setName("test_catalog") + .setId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), + catalogConfig.serialize()) + .build(); + + // storageName intentionally omitted (null) — namespace defines a config but uses the default + // AWS credential chain rather than a named storage credential. + PolarisStorageConfigurationInfo nsConfig = + AwsStorageConfigurationInfo.builder() + .addAllowedLocation("s3://ns-bucket/") + .roleARN("arn:aws:iam::123456789012:role/ns-role") + .region("us-east-1") + .build(); + + NamespaceEntity namespace = + new NamespaceEntity.Builder(Namespace.of("namespace1")) + .setCatalogId(1L) + .setId(10L) + .setParentId(1L) + .addInternalProperty( + PolarisEntityConstants.getStorageConfigInfoPropertyName(), nsConfig.serialize()) + .build(); + + IcebergTableLikeEntity table = + new IcebergTableLikeEntity.Builder( + PolarisEntitySubType.ICEBERG_TABLE, + TableIdentifier.of("namespace1", "table1"), + "s3://ns-bucket/namespace1/table1/metadata/v1.metadata.json") + .setCatalogId(1L) + .setId(100L) + .setParentId(10L) + .build(); + + PolarisResolvedPathWrapper resolvedPath = + new PolarisResolvedPathWrapper( + List.of( + resolved(PolarisEntity.of(catalog)), + resolved(PolarisEntity.of(namespace)), + resolved(PolarisEntity.of(table)))); + + Optional result = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + assertThat(result) + .as("Hierarchy walk must find a storage config (namespace has one, even with null name)") + .isPresent(); + assertThat(result.get().getId()) + .as( + "Walk must stop at the namespace (id=10), NOT fall through to the catalog (id=1) just " + + "because storageName is null") + .isEqualTo(10L); + + PolarisStorageConfigurationInfo foundConfig = + PolarisStorageConfigurationInfo.deserialize( + result + .get() + .getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getStorageConfigInfoPropertyName())); + + assertThat(foundConfig.getStorageName()) + .as( + "storageName must be null — the credential provider will use the default AWS chain, " + + "not the catalog's 'cat-creds' named credentials") + .isNull(); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/PolarisStorageConfigIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/PolarisStorageConfigIntegrationTest.java new file mode 100644 index 0000000000..9e74940038 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/PolarisStorageConfigIntegrationTest.java @@ -0,0 +1,800 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.io; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.types.Types; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; +import org.apache.polaris.core.admin.model.NamespaceStorageConfigResponse; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.admin.model.TableStorageConfigResponse; +import org.apache.polaris.service.TestServices; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for storage configuration management endpoints. + * + *

These tests verify the Management API endpoints for: + * + *

    + *
  • GET/PUT/DELETE namespace storage config + *
  • GET/PUT/DELETE table storage config + *
+ */ +public class PolarisStorageConfigIntegrationTest { + + private static final String TEST_CATALOG = "test_storage_config_catalog"; + private static final String TEST_NAMESPACE = "test_namespace"; + private static final String TEST_TABLE = "test_table"; + private static final String STORAGE_CONFIG_SOURCE_HEADER = "X-Polaris-Storage-Config-Source"; + private static final Schema SCHEMA = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "data", Types.StringType.get())); + + private TestServices services; + + @BeforeEach + public void setup() { + services = + TestServices.builder() + .config( + Map.of( + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("S3", "GCS", "AZURE", "FILE"), + "ALLOW_INSECURE_STORAGE_TYPES", + true)) + .build(); + + // Create test catalog + FileStorageConfigInfo catalogStorageConfig = + FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .setAllowedLocations(List.of("file:///tmp/test/")) + .setStorageName("catalog-storage") + .build(); + + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(TEST_CATALOG) + .setProperties(new CatalogProperties("file:///tmp/test/")) + .setStorageConfigInfo(catalogStorageConfig) + .build(); + + try (Response response = + services + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalog), + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.CREATED.getStatusCode()); + } + + // Create test namespace + CreateNamespaceRequest createNamespaceRequest = + CreateNamespaceRequest.builder().withNamespace(Namespace.of(TEST_NAMESPACE)).build(); + + try (Response response = + services + .restApi() + .createNamespace( + TEST_CATALOG, + createNamespaceRequest, + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Create test table + CreateTableRequest createTableRequest = + CreateTableRequest.builder().withName(TEST_TABLE).withSchema(SCHEMA).build(); + + try (Response response = + services + .restApi() + .createTable( + TEST_CATALOG, + TEST_NAMESPACE, + createTableRequest, + null, + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + } + + @AfterEach + public void tearDown() { + // Cleanup if needed + } + + /** Test GET namespace storage config endpoint. */ + @Test + public void testGetNamespaceStorageConfig() { + // Test GET namespace storage config - should return catalog's config since namespace has none + Response response = + services + .catalogsApi() + .getNamespaceStorageConfig( + TEST_CATALOG, TEST_NAMESPACE, services.realmContext(), services.securityContext()); + + // Expect 200 OK with catalog storage config + assertThat(response.getStatus()) + .as("GET namespace storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config).isNotNull(); + assertThat(config.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.FILE); + assertThat(config.getStorageName()).isEqualTo("catalog-storage"); + assertThat(response.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)).isEqualTo("CATALOG"); + + response.close(); + } + + /** Test PUT namespace storage config endpoint. */ + @Test + public void testSetNamespaceStorageConfig() { + // Create namespace-specific Azure storage config + AzureStorageConfigInfo namespaceStorageConfig = + AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://container@storage.dfs.core.windows.net/")) + .setStorageName("namespace-storage") + .setTenantId("tenant-123") + .build(); + + // Test PUT namespace storage config + Response response = + services + .catalogsApi() + .setNamespaceStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + namespaceStorageConfig, + services.realmContext(), + services.securityContext()); + + // Expect 200 OK + assertThat(response.getStatus()) + .as("PUT namespace storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + + NamespaceStorageConfigResponse nsResponse = + response.readEntity(NamespaceStorageConfigResponse.class); + assertThat(nsResponse).isNotNull(); + assertThat(nsResponse.getStorageConfigInfo()).isNotNull(); + assertThat(nsResponse.getStorageConfigInfo().getStorageType()) + .isEqualTo(StorageConfigInfo.StorageTypeEnum.AZURE); + + response.close(); + + // Verify we can GET it back + try (Response getResponse = + services + .catalogsApi() + .getNamespaceStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + services.realmContext(), + services.securityContext())) { + assertThat(getResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo retrieved = getResponse.readEntity(StorageConfigInfo.class); + assertThat(retrieved).isInstanceOf(AzureStorageConfigInfo.class); + AzureStorageConfigInfo azureRetrieved = (AzureStorageConfigInfo) retrieved; + assertThat(azureRetrieved.getTenantId()).isEqualTo("tenant-123"); + assertThat(azureRetrieved.getStorageName()).isEqualTo("namespace-storage"); + assertThat(getResponse.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)).isEqualTo("NAMESPACE"); + } + } + + /** Test DELETE namespace storage config endpoint. */ + @Test + public void testDeleteNamespaceStorageConfig() { + // First set a config + AzureStorageConfigInfo namespaceStorageConfig = + AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://container@storage.dfs.core.windows.net/")) + .setTenantId("tenant-123") + .build(); + + try (Response setResponse = + services + .catalogsApi() + .setNamespaceStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + namespaceStorageConfig, + services.realmContext(), + services.securityContext())) { + assertThat(setResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Now delete it + Response response = + services + .catalogsApi() + .deleteNamespaceStorageConfig( + TEST_CATALOG, TEST_NAMESPACE, services.realmContext(), services.securityContext()); + + // Expect 204 No Content + assertThat(response.getStatus()) + .as("DELETE namespace storage config should return 204") + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + + response.close(); + + // Verify it's gone - should now return catalog config + try (Response getResponse = + services + .catalogsApi() + .getNamespaceStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + services.realmContext(), + services.securityContext())) { + assertThat(getResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo retrieved = getResponse.readEntity(StorageConfigInfo.class); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.FILE); + assertThat(getResponse.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)).isEqualTo("CATALOG"); + } + } + + /** Test GET table storage config endpoint. */ + @Test + public void testGetTableStorageConfig() { + // Test GET table storage config - should return catalog's config since table has none + Response response = + services + .catalogsApi() + .getTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext()); + + // Expect 200 OK + assertThat(response.getStatus()) + .as("GET table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + + StorageConfigInfo config = response.readEntity(StorageConfigInfo.class); + assertThat(config).isNotNull(); + assertThat(config.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.FILE); + assertThat(response.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)).isEqualTo("CATALOG"); + + response.close(); + } + + /** Test PUT table storage config endpoint. */ + @Test + public void testSetTableStorageConfig() { + // Create table-specific S3 storage config + AwsStorageConfigInfo tableStorageConfig = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://table-bucket/")) + .setStorageName("table-storage") + .setRoleArn("arn:aws:iam::123456789012:role/table-role") + .build(); + + // Test PUT table storage config + Response response = + services + .catalogsApi() + .setTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + tableStorageConfig, + services.realmContext(), + services.securityContext()); + + // Expect 200 OK + assertThat(response.getStatus()) + .as("PUT table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + + TableStorageConfigResponse tableResponse = + response.readEntity(TableStorageConfigResponse.class); + assertThat(tableResponse).isNotNull(); + assertThat(tableResponse.getStorageConfigInfo()).isNotNull(); + assertThat(tableResponse.getStorageConfigInfo().getStorageType()) + .isEqualTo(StorageConfigInfo.StorageTypeEnum.S3); + + response.close(); + + // Verify we can GET it back + try (Response getResponse = + services + .catalogsApi() + .getTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext())) { + assertThat(getResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo retrieved = getResponse.readEntity(StorageConfigInfo.class); + assertThat(retrieved).isInstanceOf(AwsStorageConfigInfo.class); + AwsStorageConfigInfo awsRetrieved = (AwsStorageConfigInfo) retrieved; + assertThat(awsRetrieved.getRoleArn()).isEqualTo("arn:aws:iam::123456789012:role/table-role"); + assertThat(awsRetrieved.getStorageName()).isEqualTo("table-storage"); + assertThat(getResponse.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)).isEqualTo("TABLE"); + } + } + + @Test + public void testTableEffectiveFallbackSemantics() { + AzureStorageConfigInfo namespaceStorageConfig = + AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://container@storage.dfs.core.windows.net/")) + .setStorageName("namespace-effective") + .setTenantId("tenant-effective") + .build(); + + AwsStorageConfigInfo tableStorageConfig = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://table-effective/")) + .setStorageName("table-effective") + .setRoleArn("arn:aws:iam::123456789012:role/table-effective") + .build(); + + try (Response setNamespaceResponse = + services + .catalogsApi() + .setNamespaceStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + namespaceStorageConfig, + services.realmContext(), + services.securityContext())) { + assertThat(setNamespaceResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response setTableResponse = + services + .catalogsApi() + .setTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + tableStorageConfig, + services.realmContext(), + services.securityContext())) { + assertThat(setTableResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response getTableResponse = + services + .catalogsApi() + .getTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext())) { + assertThat(getTableResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AwsStorageConfigInfo tableConfig = getTableResponse.readEntity(AwsStorageConfigInfo.class); + assertThat(tableConfig.getStorageName()).isEqualTo("table-effective"); + assertThat(getTableResponse.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)).isEqualTo("TABLE"); + } + + try (Response deleteTableResponse = + services + .catalogsApi() + .deleteTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext())) { + assertThat(deleteTableResponse.getStatus()) + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response getNamespaceFallback = + services + .catalogsApi() + .getTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext())) { + assertThat(getNamespaceFallback.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AzureStorageConfigInfo namespaceConfig = + getNamespaceFallback.readEntity(AzureStorageConfigInfo.class); + assertThat(namespaceConfig.getStorageName()).isEqualTo("namespace-effective"); + assertThat(getNamespaceFallback.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)) + .isEqualTo("NAMESPACE"); + } + + try (Response deleteNamespaceResponse = + services + .catalogsApi() + .deleteNamespaceStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + services.realmContext(), + services.securityContext())) { + assertThat(deleteNamespaceResponse.getStatus()) + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + } + + try (Response getCatalogFallback = + services + .catalogsApi() + .getTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext())) { + assertThat(getCatalogFallback.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + FileStorageConfigInfo catalogConfig = + getCatalogFallback.readEntity(FileStorageConfigInfo.class); + assertThat(catalogConfig.getStorageName()).isEqualTo("catalog-storage"); + assertThat(getCatalogFallback.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)) + .isEqualTo("CATALOG"); + } + } + + @Test + public void testTableEffectiveFallbackUsesAncestorNamespaceInNestedPath() { + String namespaceL1 = "ns1"; + String namespaceL3 = "ns1\u001Fns2\u001Fns3"; + String nestedTable = "nested_table"; + + try (Response createNs1 = + services + .restApi() + .createNamespace( + TEST_CATALOG, + CreateNamespaceRequest.builder().withNamespace(Namespace.of("ns1")).build(), + services.realmContext(), + services.securityContext())) { + assertThat(createNs1.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response createNs2 = + services + .restApi() + .createNamespace( + TEST_CATALOG, + CreateNamespaceRequest.builder().withNamespace(Namespace.of("ns1", "ns2")).build(), + services.realmContext(), + services.securityContext())) { + assertThat(createNs2.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response createNs3 = + services + .restApi() + .createNamespace( + TEST_CATALOG, + CreateNamespaceRequest.builder() + .withNamespace(Namespace.of("ns1", "ns2", "ns3")) + .build(), + services.realmContext(), + services.securityContext())) { + assertThat(createNs3.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response createNestedTable = + services + .restApi() + .createTable( + TEST_CATALOG, + namespaceL3, + CreateTableRequest.builder().withName(nestedTable).withSchema(SCHEMA).build(), + null, + services.realmContext(), + services.securityContext())) { + assertThat(createNestedTable.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + AzureStorageConfigInfo ancestorStorageConfig = + AzureStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .setAllowedLocations(List.of("abfss://ancestor@storage.dfs.core.windows.net/")) + .setStorageName("ancestor-storage") + .setTenantId("ancestor-tenant") + .build(); + + try (Response setAncestorConfig = + services + .catalogsApi() + .setNamespaceStorageConfig( + TEST_CATALOG, + namespaceL1, + ancestorStorageConfig, + services.realmContext(), + services.securityContext())) { + assertThat(setAncestorConfig.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response getNestedTableConfig = + services + .catalogsApi() + .getTableStorageConfig( + TEST_CATALOG, + namespaceL3, + nestedTable, + services.realmContext(), + services.securityContext())) { + assertThat(getNestedTableConfig.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + AzureStorageConfigInfo resolved = + getNestedTableConfig.readEntity(AzureStorageConfigInfo.class); + assertThat(resolved.getStorageName()).isEqualTo("ancestor-storage"); + assertThat(resolved.getTenantId()).isEqualTo("ancestor-tenant"); + assertThat(getNestedTableConfig.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)) + .isEqualTo("NAMESPACE"); + } + } + + /** Test DELETE table storage config endpoint. */ + @Test + public void testDeleteTableStorageConfig() { + // First set a config + AwsStorageConfigInfo tableStorageConfig = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://table-bucket/")) + .setRoleArn("arn:aws:iam::123456789012:role/table-role") + .build(); + + try (Response setResponse = + services + .catalogsApi() + .setTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + tableStorageConfig, + services.realmContext(), + services.securityContext())) { + assertThat(setResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Now delete it + Response response = + services + .catalogsApi() + .deleteTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext()); + + // Expect 204 No Content + assertThat(response.getStatus()) + .as("DELETE table storage config should return 204") + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); + + response.close(); + + // Verify it's gone - should now return catalog config + try (Response getResponse = + services + .catalogsApi() + .getTableStorageConfig( + TEST_CATALOG, + TEST_NAMESPACE, + TEST_TABLE, + services.realmContext(), + services.securityContext())) { + assertThat(getResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo retrieved = getResponse.readEntity(StorageConfigInfo.class); + assertThat(retrieved.getStorageType()).isEqualTo(StorageConfigInfo.StorageTypeEnum.FILE); + assertThat(getResponse.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)).isEqualTo("CATALOG"); + } + } + + /** Test storage config operations with multipart namespace paths. */ + @Test + public void testMultipartNamespaceStorageConfig() { + // Create a multipart namespace - need to create parents first + String multipartNamespace = "level1\u001Flevel2\u001Flevel3"; // Unit separator + + // Create parent namespaces first + CreateNamespaceRequest createLevel1Request = + CreateNamespaceRequest.builder().withNamespace(Namespace.of("level1")).build(); + + try (Response createResponse = + services + .restApi() + .createNamespace( + TEST_CATALOG, + createLevel1Request, + services.realmContext(), + services.securityContext())) { + assertThat(createResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + CreateNamespaceRequest createLevel2Request = + CreateNamespaceRequest.builder().withNamespace(Namespace.of("level1", "level2")).build(); + + try (Response createResponse = + services + .restApi() + .createNamespace( + TEST_CATALOG, + createLevel2Request, + services.realmContext(), + services.securityContext())) { + assertThat(createResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + CreateNamespaceRequest createNamespaceRequest = + CreateNamespaceRequest.builder() + .withNamespace(Namespace.of("level1", "level2", "level3")) + .build(); + + try (Response createResponse = + services + .restApi() + .createNamespace( + TEST_CATALOG, + createNamespaceRequest, + services.realmContext(), + services.securityContext())) { + assertThat(createResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Create a GCP storage config for multipart namespace + GcpStorageConfigInfo storageConfig = + GcpStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS) + .setAllowedLocations(List.of("gs://gcs-bucket/path/")) + .setGcsServiceAccount("test@test.iam.gserviceaccount.com") + .build(); + + // Test with multipart namespace + Response response = + services + .catalogsApi() + .setNamespaceStorageConfig( + TEST_CATALOG, + multipartNamespace, + storageConfig, + services.realmContext(), + services.securityContext()); + + // Expect 200 OK + assertThat(response.getStatus()) + .as("Multipart namespace storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + + response.close(); + + // Verify we can GET it back + try (Response getResponse = + services + .catalogsApi() + .getNamespaceStorageConfig( + TEST_CATALOG, + multipartNamespace, + services.realmContext(), + services.securityContext())) { + assertThat(getResponse.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + StorageConfigInfo retrieved = getResponse.readEntity(StorageConfigInfo.class); + assertThat(retrieved).isInstanceOf(GcpStorageConfigInfo.class); + GcpStorageConfigInfo gcpRetrieved = (GcpStorageConfigInfo) retrieved; + assertThat(gcpRetrieved.getGcsServiceAccount()) + .isEqualTo("test@test.iam.gserviceaccount.com"); + } + } + + /** + * Regression test for nested-namespace parent fallback in the namespace GET endpoint. + * + *

Verifies that when a deeply nested namespace (e.g. {@code ns1\u001Fns2\u001Fns3}) has no + * inline storage config, the effective GET response walks up to the nearest ancestor that does + * have one — rather than jumping directly to the catalog. This exercises the fixed path through + * {@code resolveEffectiveNamespaceStorageConfig} which now uses {@code + * PolarisAdminService.resolveNamespacePath} + {@code FileIOUtil.findStorageInfoFromHierarchy}. + */ + @Test + public void testGetNamespaceStorageConfigFallsBackToAncestorNamespace() { + // Arrange: create ns1 → ns1.ns2 → ns1.ns2.ns3 hierarchy + for (Namespace ns : + List.of( + Namespace.of("ns1"), Namespace.of("ns1", "ns2"), Namespace.of("ns1", "ns2", "ns3"))) { + try (Response r = + services + .restApi() + .createNamespace( + TEST_CATALOG, + CreateNamespaceRequest.builder().withNamespace(ns).build(), + services.realmContext(), + services.securityContext())) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + } + + // Place a storage config only on ns1 (the root ancestor) + AwsStorageConfigInfo ns1Config = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://ns1-bucket/")) + .setRoleArn("arn:aws:iam::111111111111:role/ns1-role") + .setStorageName("ns1-storage") + .build(); + try (Response r = + services + .catalogsApi() + .setNamespaceStorageConfig( + TEST_CATALOG, + "ns1", + ns1Config, + services.realmContext(), + services.securityContext())) { + assertThat(r.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + } + + // Act: GET effective config for the leaf namespace ns1.ns2.ns3 (has no direct config) + try (Response response = + services + .catalogsApi() + .getNamespaceStorageConfig( + TEST_CATALOG, + "ns1\u001Fns2\u001Fns3", + services.realmContext(), + services.securityContext())) { + assertThat(response.getStatus()) + .as("Leaf namespace with no config should find ancestor's config") + .isEqualTo(Response.Status.OK.getStatusCode()); + + AwsStorageConfigInfo resolved = response.readEntity(AwsStorageConfigInfo.class); + assertThat(resolved.getStorageName()) + .as("Config must come from ns1, not the catalog") + .isEqualTo("ns1-storage"); + assertThat(resolved.getRoleArn()).isEqualTo("arn:aws:iam::111111111111:role/ns1-role"); + // The source header must say NAMESPACE, not CATALOG, confirming the walk stopped at ns1 + assertThat(response.getHeaderString(STORAGE_CONFIG_SOURCE_HEADER)) + .as("Config source must be NAMESPACE (ns1), not CATALOG") + .isEqualTo("NAMESPACE"); + } + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/StorageNameHierarchyCredentialTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/StorageNameHierarchyCredentialTest.java new file mode 100644 index 0000000000..44aafe27a1 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/StorageNameHierarchyCredentialTest.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.io; + +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.types.Types; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.TestServices; +import org.junit.jupiter.api.Test; + +/** + * Service-layer tests verifying that {@code RESOLVE_CREDENTIALS_BY_STORAGE_NAME} and the + * hierarchical storage-config resolver interact correctly. + * + *

Specifically: when a namespace entity carries a {@code storageName} and a table in that + * namespace has no own storage config, the hierarchy resolver must return the namespace entity — + * not the catalog — so that {@code PolarisStorageIntegrationProviderImpl} dispatches to the correct + * named credential set. + * + *

NOTE: The {@code TestServices} in-memory layer does not invoke real AWS STS; therefore these + * tests assert the observable {@code storageName} on the effective storage config returned by the + * management API (GET table storage config), which is the direct precondition for {@code + * PolarisStorageIntegrationProviderImpl} to dispatch correctly. End-to-end STS-credential + * assertions are covered in the full-HTTP integration tests in {@code + * integration-tests/.../PolarisStorageConfigIntegrationTest.java} (Tasks 3–5). + */ +public class StorageNameHierarchyCredentialTest { + + private static final String CATALOG = "cred_test_catalog"; + private static final String NAMESPACE = "cred_ns"; + private static final String TABLE = "cred_table"; + private static final Schema SCHEMA = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "data", Types.StringType.get())); + + /** + * Build a {@link TestServices} with the given feature-flag value for {@code + * RESOLVE_CREDENTIALS_BY_STORAGE_NAME} and bootstrap a catalog + namespace + table. Returns the + * configured service instance. + */ + private TestServices buildServices(boolean resolveByStorageName) { + TestServices services = + TestServices.builder() + .config( + Map.of( + "RESOLVE_CREDENTIALS_BY_STORAGE_NAME", + resolveByStorageName, + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("S3", "FILE"), + "ALLOW_INSECURE_STORAGE_TYPES", + true)) + .build(); + + // Catalog with storageName="cat-creds" + FileStorageConfigInfo catalogStorage = + FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .setAllowedLocations(List.of("file:///tmp/test/")) + .setStorageName("cat-creds") + .build(); + + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(CATALOG) + .setProperties(new CatalogProperties("file:///tmp/test/")) + .setStorageConfigInfo(catalogStorage) + .build(); + + try (Response r = + services + .catalogsApi() + .createCatalog( + new CreateCatalogRequest(catalog), + services.realmContext(), + services.securityContext())) { + assertThat(r.getStatus()) + .as("Catalog creation should succeed") + .isEqualTo(Response.Status.CREATED.getStatusCode()); + } + + try (Response r = + services + .restApi() + .createNamespace( + CATALOG, + CreateNamespaceRequest.builder().withNamespace(Namespace.of(NAMESPACE)).build(), + services.realmContext(), + services.securityContext())) { + assertThat(r.getStatus()) + .as("Namespace creation should succeed") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + try (Response r = + services + .restApi() + .createTable( + CATALOG, + NAMESPACE, + CreateTableRequest.builder().withName(TABLE).withSchema(SCHEMA).build(), + null, + services.realmContext(), + services.securityContext())) { + assertThat(r.getStatus()) + .as("Table creation should succeed") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + return services; + } + + /** + * When {@code RESOLVE_CREDENTIALS_BY_STORAGE_NAME=true} and the namespace has a storage config + * with {@code storageName="ns-creds"}, the effective (resolved) config for a table in that + * namespace must carry {@code storageName="ns-creds"} — not the catalog's {@code + * storageName="cat-creds"}. + * + *

This is the key pre-condition for {@code PolarisStorageIntegrationProviderImpl} to dispatch + * {@code stsCredentials("ns-creds")} rather than {@code stsCredentials("cat-creds")} when the + * flag is enabled. + */ + @Test + public void testNamespaceStorageNameReachesCredentialProviderWhenFlagEnabled() { + TestServices services = buildServices(true); + + // PUT namespace storage config with storageName="ns-creds" + AwsStorageConfigInfo nsStorage = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://ns-bucket/")) + .setRoleArn("arn:aws:iam::123456789012:role/ns-role") + .setStorageName("ns-creds") + .build(); + + try (Response r = + services + .catalogsApi() + .setNamespaceStorageConfig( + CATALOG, + NAMESPACE, + nsStorage, + services.realmContext(), + services.securityContext())) { + assertThat(r.getStatus()) + .as("Setting namespace storage config should succeed") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + // GET table effective storage config — must resolve to the namespace entity, not the catalog + try (Response r = + services + .catalogsApi() + .getTableStorageConfig( + CATALOG, NAMESPACE, TABLE, services.realmContext(), services.securityContext())) { + assertThat(r.getStatus()) + .as("GET effective table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + + StorageConfigInfo effective = r.readEntity(StorageConfigInfo.class); + assertThat(effective).isNotNull(); + + // The effective storageName must be "ns-creds" (from namespace), not "cat-creds" (catalog). + // PolarisStorageIntegrationProviderImpl will call stsCredentials("ns-creds") when + // RESOLVE_CREDENTIALS_BY_STORAGE_NAME=true, giving the correct isolated credential set. + assertThat(effective.getStorageName()) + .as( + "Effective storageName must be 'ns-creds' from the namespace, not 'cat-creds' " + + "from the catalog — this is the direct precondition for the credential " + + "provider to dispatch to the namespace-scoped credential set") + .isEqualTo("ns-creds"); + assertThat(effective.getStorageName()) + .as("Catalog storageName 'cat-creds' must not leak into the effective config") + .isNotEqualTo("cat-creds"); + } + } + + /** + * When {@code RESOLVE_CREDENTIALS_BY_STORAGE_NAME=false} (the default), the catalog-level storage + * config is the fallback for credential-provider dispatch. Even if a namespace config with a + * different {@code storageName} exists, the flag being off means the provider will use the + * default AWS credential chain — but the hierarchy still resolves to the most-specific entity. + * + *

This test verifies that the storageName on the namespace is still returned in the effective + * config when the namespace has a config (the hierarchy walk is independent of the flag), while + * documenting that the flag itself controls whether the storageName is *acted upon* at the + * credential-provider layer. + */ + @Test + public void testHierarchyAlwaysResolvesNamespaceStorageNameRegardlessOfFlag() { + // RESOLVE_CREDENTIALS_BY_STORAGE_NAME=false — flag off + TestServices services = buildServices(false); + + // PUT namespace storage config with storageName="ns-creds" + AwsStorageConfigInfo nsStorage = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://ns-bucket/")) + .setRoleArn("arn:aws:iam::123456789012:role/ns-role") + .setStorageName("ns-creds") + .build(); + + try (Response r = + services + .catalogsApi() + .setNamespaceStorageConfig( + CATALOG, + NAMESPACE, + nsStorage, + services.realmContext(), + services.securityContext())) { + assertThat(r.getStatus()) + .as("Setting namespace storage config should succeed") + .isEqualTo(Response.Status.OK.getStatusCode()); + } + + // The hierarchy ALWAYS resolves to the most-specific entity regardless of the flag. + // With the flag off, PolarisStorageIntegrationProviderImpl ignores storageName and uses the + // default AWS credential chain — but the resolved storageName is still "ns-creds" in the + // entity that would be passed to the provider. + try (Response r = + services + .catalogsApi() + .getTableStorageConfig( + CATALOG, NAMESPACE, TABLE, services.realmContext(), services.securityContext())) { + assertThat(r.getStatus()) + .as("GET effective table storage config should return 200") + .isEqualTo(Response.Status.OK.getStatusCode()); + + StorageConfigInfo effective = r.readEntity(StorageConfigInfo.class); + assertThat(effective).isNotNull(); + // The hierarchy still stops at the namespace regardless of the flag. + assertThat(effective.getStorageName()) + .as( + "Hierarchy always resolves to the namespace-level storageName; the flag only " + + "controls whether the credential provider acts on it") + .isEqualTo("ns-creds"); + } + } +} diff --git a/runtime/spark-tests/build.gradle.kts b/runtime/spark-tests/build.gradle.kts index 78d21ed5a6..c15b406c84 100644 --- a/runtime/spark-tests/build.gradle.kts +++ b/runtime/spark-tests/build.gradle.kts @@ -72,7 +72,38 @@ tasks.named("intTest").configure { systemProperty("java.security.manager", "allow") // Same issue as above: allow a java security manager after Java 21 // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") + // Optional: enable additional storage-resolution overrides only for targeted hierarchy tests. + val enableStorageHierarchyOverrides = + providers.gradleProperty("enableStorageHierarchyOverrides").orNull == "true" + + val testArgLine = buildString { + append("-Djava.security.manager=allow") + + if (enableStorageHierarchyOverrides) { + append(" -Dpolaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"=true") + + // Named storages used by Spark storage hierarchy integration tests. + listOf( + "validstorage", + "ns-named", + "tbl-named", + "ns", + "tbl", + "billing-creds", + "mid", + "tbl-only", + "ns-shared", + "tbl-override", + "ns-v1", + "ns-v2", + ) + .forEach { storageName -> + append(" -Dpolaris.storage.aws.$storageName.access-key=foo") + append(" -Dpolaris.storage.aws.$storageName.secret-key=bar") + } + } + } + systemProperty("quarkus.test.arg-line", testArgLine) val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") // delete files from previous runs doFirst { diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/RealVendingProfile.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/RealVendingProfile.java new file mode 100644 index 0000000000..bb2f2b6d8e --- /dev/null +++ b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/RealVendingProfile.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.Map; + +public class RealVendingProfile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.ofEntries( + Map.entry("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false"), + Map.entry("polaris.storage.aws.access-key", "foo"), + Map.entry("polaris.storage.aws.secret-key", "bar"), + Map.entry("polaris.storage.aws.validstorage.access-key", "foo"), + Map.entry("polaris.storage.aws.validstorage.secret-key", "bar"), + Map.entry("polaris.storage.aws.ns-named.access-key", "foo"), + Map.entry("polaris.storage.aws.ns-named.secret-key", "bar"), + Map.entry("polaris.storage.aws.tbl-named.access-key", "foo"), + Map.entry("polaris.storage.aws.tbl-named.secret-key", "bar"), + Map.entry("polaris.storage.aws.ns.access-key", "foo"), + Map.entry("polaris.storage.aws.ns.secret-key", "bar"), + Map.entry("polaris.storage.aws.tbl.access-key", "foo"), + Map.entry("polaris.storage.aws.tbl.secret-key", "bar"), + Map.entry("polaris.storage.aws.billing-creds.access-key", "foo"), + Map.entry("polaris.storage.aws.billing-creds.secret-key", "bar"), + Map.entry("polaris.storage.aws.mid.access-key", "foo"), + Map.entry("polaris.storage.aws.mid.secret-key", "bar"), + Map.entry("polaris.storage.aws.tbl-only.access-key", "foo"), + Map.entry("polaris.storage.aws.tbl-only.secret-key", "bar"), + Map.entry("polaris.storage.aws.ns-shared.access-key", "foo"), + Map.entry("polaris.storage.aws.ns-shared.secret-key", "bar"), + Map.entry("polaris.storage.aws.tbl-override.access-key", "foo"), + Map.entry("polaris.storage.aws.tbl-override.secret-key", "bar"), + Map.entry("polaris.storage.aws.ns-v1.access-key", "foo"), + Map.entry("polaris.storage.aws.ns-v1.secret-key", "bar"), + Map.entry("polaris.storage.aws.ns-v2.access-key", "foo"), + Map.entry("polaris.storage.aws.ns-v2.secret-key", "bar")); + } +} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageConfigCredentialVendingIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageConfigCredentialVendingIT.java new file mode 100644 index 0000000000..645dfd9670 --- /dev/null +++ b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageConfigCredentialVendingIT.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import org.apache.polaris.service.it.test.PolarisSparkStorageConfigCredentialVendingIntegrationTest; + +@QuarkusIntegrationTest +public class SparkStorageConfigCredentialVendingIT + extends PolarisSparkStorageConfigCredentialVendingIntegrationTest {} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageConfigCredentialVendingRealIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageConfigCredentialVendingRealIT.java new file mode 100644 index 0000000000..d92769ae28 --- /dev/null +++ b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageConfigCredentialVendingRealIT.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisSparkStorageConfigCredentialVendingRealIntegrationTest; + +@QuarkusIntegrationTest +@TestProfile(RealVendingProfile.class) +public class SparkStorageConfigCredentialVendingRealIT + extends PolarisSparkStorageConfigCredentialVendingRealIntegrationTest {} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageNameHierarchyIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageNameHierarchyIT.java new file mode 100644 index 0000000000..c06fc3bc19 --- /dev/null +++ b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageNameHierarchyIT.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import org.apache.polaris.service.it.test.PolarisSparkStorageNameHierarchyIntegrationTest; + +@QuarkusIntegrationTest +public class SparkStorageNameHierarchyIT extends PolarisSparkStorageNameHierarchyIntegrationTest {} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageNameHierarchyRealIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageNameHierarchyRealIT.java new file mode 100644 index 0000000000..c4c75bd78a --- /dev/null +++ b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkStorageNameHierarchyRealIT.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisSparkStorageNameHierarchyRealIntegrationTest; + +@QuarkusIntegrationTest +@TestProfile(RealVendingProfile.class) +public class SparkStorageNameHierarchyRealIT + extends PolarisSparkStorageNameHierarchyRealIntegrationTest {} diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index de53178728..f22b4bd849 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -806,6 +806,173 @@ paths: 404: description: "The catalog or the role does not exist" + /catalogs/{catalogName}/namespaces/{namespace}/storage-config: + parameters: + - name: catalogName + in: path + description: The name of the catalog + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: namespace + in: path + description: | + A namespace identifier as a single string. + Multipart namespace parts should be separated by the unit separator (`0x1F`) byte. + required: true + schema: + type: string + examples: + singlepart_namespace: + value: "accounting" + multipart_namespace: + value: "accounting%1Ftax" + get: + operationId: getNamespaceStorageConfig + summary: Get effective storage configuration for namespace + description: | + Returns the effective storage configuration for the namespace. + This is either the namespace-specific override (if set) or the catalog-wide config. + responses: + 200: + description: Storage configuration retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/StorageConfigInfo' + 403: + description: "The caller does not have permission to read namespace storage configuration" + 404: + description: "The namespace or catalog does not exist" + put: + operationId: setNamespaceStorageConfig + summary: Set or update namespace storage configuration override + description: | + Sets a namespace-specific storage configuration override. + This will take precedence over the catalog-wide config for operations + on tables within this namespace (unless the table has its own override). + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/StorageConfigInfo' + responses: + 200: + description: Configuration updated successfully + content: + application/json: + schema: + $ref: '#/components/schemas/NamespaceStorageConfigResponse' + 400: + description: "Invalid storage configuration" + 403: + description: "The caller does not have permission to modify namespace storage configuration" + 404: + description: "The namespace or catalog does not exist" + delete: + operationId: deleteNamespaceStorageConfig + summary: Remove namespace storage configuration override + description: | + Removes the namespace-specific storage configuration override. + The namespace will revert to using the catalog-wide config. + responses: + 204: + description: "Configuration removed successfully" + 403: + description: "The caller does not have permission to delete namespace storage configuration" + 404: + description: "The namespace or catalog does not exist" + + /catalogs/{catalogName}/namespaces/{namespace}/tables/{table}/storage-config: + parameters: + - name: catalogName + in: path + description: The name of the catalog + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: namespace + in: path + description: | + A namespace identifier as a single string. + Multipart namespace parts should be separated by the unit separator (`0x1F`) byte. + required: true + schema: + type: string + examples: + singlepart_namespace: + value: "accounting" + multipart_namespace: + value: "accounting%1Ftax" + - name: table + in: path + description: A table name + required: true + schema: + type: string + get: + operationId: getTableStorageConfig + summary: Get effective storage configuration for table + description: | + Returns the effective storage configuration for the table. + This is resolved from the hierarchy: table override → namespace override → catalog config. + responses: + 200: + description: Storage configuration retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/StorageConfigInfo' + 403: + description: "The caller does not have permission to read table storage configuration" + 404: + description: "The table, namespace, or catalog does not exist" + put: + operationId: setTableStorageConfig + summary: Set or update table storage configuration override + description: | + Sets a table-specific storage configuration override. + This will take precedence over namespace and catalog configurations. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/StorageConfigInfo' + responses: + 200: + description: Configuration updated successfully + content: + application/json: + schema: + $ref: '#/components/schemas/TableStorageConfigResponse' + 400: + description: "Invalid storage configuration" + 403: + description: "The caller does not have permission to modify table storage configuration" + 404: + description: "The table, namespace, or catalog does not exist" + delete: + operationId: deleteTableStorageConfig + summary: Remove table storage configuration override + description: | + Removes the table-specific storage configuration override. + The table will revert to using the namespace or catalog config. + responses: + 204: + description: "Configuration removed successfully" + 403: + description: "The caller does not have permission to delete table storage configuration" + 404: + description: "The table, namespace, or catalog does not exist" + components: securitySchemes: OAuth2: @@ -1200,6 +1367,51 @@ components: allOf: - $ref: '#/components/schemas/StorageConfigInfo' + NamespaceStorageConfigResponse: + type: object + description: Response after setting or updating namespace storage configuration + properties: + namespace: + type: array + items: + type: string + description: The namespace identifier as an array of parts + example: ["accounting", "tax"] + storageConfigInfo: + $ref: '#/components/schemas/StorageConfigInfo' + message: + type: string + description: Optional message about the operation + example: "Storage configuration updated successfully" + required: + - namespace + - storageConfigInfo + + TableStorageConfigResponse: + type: object + description: Response after setting or updating table storage configuration + properties: + namespace: + type: array + items: + type: string + description: The namespace identifier as an array of parts + example: ["accounting", "tax"] + table: + type: string + description: The table name + example: "sales" + storageConfigInfo: + $ref: '#/components/schemas/StorageConfigInfo' + message: + type: string + description: Optional message about the operation + example: "Storage configuration updated successfully" + required: + - namespace + - table + - storageConfigInfo + ServiceIdentityInfo: type: object description: Identity metadata for the Polaris service used to access external resources.