From e90b501ff057cba21cc2b17e716112f266bce721 Mon Sep 17 00:00:00 2001 From: Prashant Date: Mon, 29 Sep 2025 19:42:27 -0700 Subject: [PATCH 1/6] Persistence: Handle schema evolution --- .../jdbc/JdbcBasePersistenceImpl.java | 73 +++++++++++++------ .../relational/jdbc/QueryGenerator.java | 16 +++- .../relational/jdbc/models/Converter.java | 6 +- .../EntityNameLookupRecordConverter.java | 2 +- .../relational/jdbc/models/ModelEntity.java | 44 ++++++++++- .../relational/jdbc/models/ModelEvent.java | 2 +- .../jdbc/models/ModelGrantRecord.java | 2 +- .../jdbc/models/ModelPolicyMappingRecord.java | 2 +- .../ModelPrincipalAuthenticationData.java | 2 +- .../relational/jdbc/models/SchemaVersion.java | 2 +- ...anagerWithJdbcBasePersistenceImplTest.java | 10 ++- ...thJdbcBasePersistenceImplV1SchemaTest.java | 28 +++++++ ...thJdbcBasePersistenceImplV2SchemaTest.java | 28 +++++++ ...thJdbcBasePersistenceImplV3SchemaTest.java | 28 +++++++ .../jdbc/DatasourceOperationsTest.java | 2 +- .../relational/jdbc/QueryGeneratorTest.java | 43 ++++++----- .../jdbc/models/ModelEventTest.java | 4 +- .../BasePolarisMetaStoreManagerTest.java | 2 +- .../config/ProductionReadinessChecks.java | 32 ++++++++ 19 files changed, 260 insertions(+), 68 deletions(-) create mode 100644 persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV1SchemaTest.java create mode 100644 persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV2SchemaTest.java create mode 100644 persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV3SchemaTest.java diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 5f363e3a7d..9b6cba3492 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -178,11 +178,18 @@ private void persistEntity( if (originalEntity == null) { try { List values = - modelEntity.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(); + modelEntity + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .values() + .stream() + .toList(); queryAction.apply( connection, QueryGenerator.generateInsertQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, values, realmId)); + ModelEntity.getAllColumnNames(schemaVersion), + ModelEntity.TABLE_NAME, + values, + realmId)); } catch (SQLException e) { if (datasourceOperations.isConstraintViolation(e)) { PolarisBaseEntity existingEntity = @@ -217,12 +224,19 @@ private void persistEntity( realmId); try { List values = - modelEntity.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(); + modelEntity + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .values() + .stream() + .toList(); int rowsUpdated = queryAction.apply( connection, QueryGenerator.generateUpdateQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, values, params)); + ModelEntity.getAllColumnNames(schemaVersion), + ModelEntity.TABLE_NAME, + values, + params)); if (rowsUpdated == 0) { throw new RetryOnConcurrencyException( "Entity '%s' id '%s' concurrently modified; expected version %s", @@ -241,7 +255,11 @@ public void writeToGrantRecords( ModelGrantRecord modelGrantRecord = ModelGrantRecord.fromGrantRecord(grantRec); try { List values = - modelGrantRecord.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(); + modelGrantRecord + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .values() + .stream() + .toList(); datasourceOperations.executeUpdate( QueryGenerator.generateInsertQuery( ModelGrantRecord.ALL_COLUMNS, ModelGrantRecord.TABLE_NAME, values, realmId)); @@ -264,7 +282,7 @@ public void writeEvents(@Nonnull List events) { ModelEvent.ALL_COLUMNS, ModelEvent.TABLE_NAME, ModelEvent.fromEvent(events.getFirst()) - .toMap(datasourceOperations.getDatabaseType()) + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) .values() .stream() .toList(), @@ -282,7 +300,7 @@ public void writeEvents(@Nonnull List events) { ModelEvent.ALL_COLUMNS, ModelEvent.TABLE_NAME, ModelEvent.fromEvent(event) - .toMap(datasourceOperations.getDatabaseType()) + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) .values() .stream() .toList(), @@ -322,7 +340,7 @@ public void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBa try { datasourceOperations.executeUpdate( QueryGenerator.generateDeleteQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, params)); + ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); } catch (SQLException e) { throw new RuntimeException( String.format("Failed to delete entity due to %s", e.getMessage()), e); @@ -335,7 +353,7 @@ public void deleteFromGrantRecords( ModelGrantRecord modelGrantRecord = ModelGrantRecord.fromGrantRecord(grantRec); try { Map whereClause = - modelGrantRecord.toMap(datasourceOperations.getDatabaseType()); + modelGrantRecord.toMap(datasourceOperations.getDatabaseType(), schemaVersion); whereClause.put("realm_id", realmId); datasourceOperations.executeUpdate( QueryGenerator.generateDeleteQuery( @@ -370,7 +388,7 @@ public void deleteAll(@Nonnull PolarisCallContext callCtx) { datasourceOperations.execute( connection, QueryGenerator.generateDeleteQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, params)); + ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); datasourceOperations.execute( connection, QueryGenerator.generateDeleteQuery( @@ -402,7 +420,7 @@ public PolarisBaseEntity lookupEntity( Map.of("catalog_id", catalogId, "id", entityId, "type_code", typeCode, "realm_id", realmId); return getPolarisBaseEntity( QueryGenerator.generateSelectQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, params)); + ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); } @Override @@ -426,7 +444,7 @@ public PolarisBaseEntity lookupEntityByName( realmId); return getPolarisBaseEntity( QueryGenerator.generateSelectQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, params)); + ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); } @Nullable @@ -454,7 +472,8 @@ private PolarisBaseEntity getPolarisBaseEntity(QueryGenerator.PreparedQuery quer public List lookupEntities( @Nonnull PolarisCallContext callCtx, List entityIds) { if (entityIds == null || entityIds.isEmpty()) return new ArrayList<>(); - PreparedQuery query = QueryGenerator.generateSelectQueryWithEntityIds(realmId, entityIds); + PreparedQuery query = + QueryGenerator.generateSelectQueryWithEntityIds(realmId, schemaVersion, entityIds); try { return datasourceOperations.executeSelect(query, new ModelEntity()); } catch (SQLException e) { @@ -575,7 +594,12 @@ public Page loadEntities( try { PreparedQuery query = buildEntityQuery( - catalogId, parentId, entityType, entitySubType, pageToken, ModelEntity.ALL_COLUMNS); + catalogId, + parentId, + entityType, + entitySubType, + pageToken, + ModelEntity.getAllColumnNames(schemaVersion)); AtomicReference> results = new AtomicReference<>(); datasourceOperations.executeSelectOverStream( query, @@ -600,7 +624,7 @@ public int lookupEntityGrantRecordsVersion( PolarisBaseEntity b = getPolarisBaseEntity( QueryGenerator.generateSelectQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, params)); + ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); return b == null ? 0 : b.getGrantRecordsVersion(); } @@ -714,7 +738,7 @@ public boolean hasChildren( var results = datasourceOperations.executeSelect( QueryGenerator.generateSelectQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, params), + ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params), new ModelEntity()); return results != null && !results.isEmpty(); } catch (SQLException e) { @@ -759,7 +783,7 @@ Optional> hasOverlappingSiblings( PreparedQuery query = QueryGenerator.generateOverlapQuery( - realmId, entity.getCatalogId(), entity.getBaseLocation()); + realmId, schemaVersion, entity.getCatalogId(), entity.getBaseLocation()); try { var results = datasourceOperations.executeSelect(query, new ModelEntity()); if (!results.isEmpty()) { @@ -835,7 +859,10 @@ public PolarisPrincipalSecrets generateNewPrincipalSecrets( // write new principal secrets try { List values = - lookupPrincipalSecrets.toMap(datasourceOperations.getDatabaseType()).values().stream() + lookupPrincipalSecrets + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .values() + .stream() .toList(); datasourceOperations.executeUpdate( QueryGenerator.generateInsertQuery( @@ -875,7 +902,7 @@ public PolarisPrincipalSecrets storePrincipalSecrets( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, modelPrincipalAuthenticationData - .toMap(datasourceOperations.getDatabaseType()) + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) .values() .stream() .toList(), @@ -936,7 +963,7 @@ public PolarisPrincipalSecrets rotatePrincipalSecrets( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, modelPrincipalAuthenticationData - .toMap(datasourceOperations.getDatabaseType()) + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) .values() .stream() .toList(), @@ -990,7 +1017,7 @@ public void writeToPolicyMappingRecords( ModelPolicyMappingRecord.fromPolicyMappingRecord(record); List values = modelPolicyMappingRecord - .toMap(datasourceOperations.getDatabaseType()) + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) .values() .stream() .toList(); @@ -1054,7 +1081,7 @@ private boolean handleInheritablePolicy( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, modelPolicyMappingRecord - .toMap(datasourceOperations.getDatabaseType()) + .toMap(datasourceOperations.getDatabaseType(), schemaVersion) .values() .stream() .toList(), @@ -1073,7 +1100,7 @@ public void deleteFromPolicyMappingRecords( var modelPolicyMappingRecord = ModelPolicyMappingRecord.fromPolicyMappingRecord(record); try { Map objectMap = - modelPolicyMappingRecord.toMap(datasourceOperations.getDatabaseType()); + modelPolicyMappingRecord.toMap(datasourceOperations.getDatabaseType(), schemaVersion); objectMap.put("realm_id", realmId); datasourceOperations.executeUpdate( QueryGenerator.generateDeleteQuery( diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java index fac862ccd2..94f6248700 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java @@ -113,12 +113,13 @@ public static PreparedQuery generateDeleteQueryForEntityGrantRecords( * Builds a SELECT query using a list of entity ID pairs (catalog_id, id). * * @param realmId Realm to filter by. + * @param schemaVersion The schema version of entities table to query * @param entityIds List of PolarisEntityId pairs. * @return SELECT query to retrieve matching entities. * @throws IllegalArgumentException if entityIds is empty. */ public static PreparedQuery generateSelectQueryWithEntityIds( - @Nonnull String realmId, @Nonnull List entityIds) { + @Nonnull String realmId, int schemaVersion, @Nonnull List entityIds) { if (entityIds.isEmpty()) { throw new IllegalArgumentException("Empty entity ids"); } @@ -131,7 +132,9 @@ public static PreparedQuery generateSelectQueryWithEntityIds( params.add(realmId); String where = " WHERE (catalog_id, id) IN (" + placeholders + ") AND realm_id = ?"; return new PreparedQuery( - generateSelectQuery(ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, where, null).sql(), + generateSelectQuery( + ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, where, null) + .sql(), params); } @@ -260,13 +263,14 @@ static PreparedQuery generateVersionQuery() { * This should be combined with a check using `StorageLocation`. * * @param realmId A realm to search within + * @param schemaVersion The schema version of entities table to query * @param catalogId A catalog entity to search within * @param baseLocation The base location to look for overlap with, with or without a scheme * @return The list of possibly overlapping entities that meet the criteria */ @VisibleForTesting public static PreparedQuery generateOverlapQuery( - String realmId, long catalogId, String baseLocation) { + String realmId, int schemaVersion, long catalogId, String baseLocation) { StorageLocation baseStorageLocation = StorageLocation.of(baseLocation); String locationWithoutScheme = baseStorageLocation.withoutScheme(); @@ -297,7 +301,11 @@ public static PreparedQuery generateOverlapQuery( QueryFragment where = new QueryFragment(clause, finalParams); PreparedQuery query = - generateSelectQuery(ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, where.sql(), null); + generateSelectQuery( + ModelEntity.getAllColumnNames(schemaVersion), + ModelEntity.TABLE_NAME, + where.sql(), + null); return new PreparedQuery(query.sql(), where.parameters()); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java index 228fcd4513..fa29a243cb 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java @@ -35,10 +35,10 @@ public interface Converter { T fromResultSet(ResultSet rs) throws SQLException; /** - * Convert a model into a Map with keys as snake case names, where as values as values of member - * of model obj. + * Convert a model into a Map with keys as snake case names, and values as values of member of + * model obj. */ - Map toMap(DatabaseType databaseType); + Map toMap(DatabaseType databaseType, int schemaVersion); default PGobject toJsonbPGobject(String props) { try { diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java index d4bd7775bb..ca34986ec9 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java @@ -39,7 +39,7 @@ public EntityNameLookupRecord fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType) { + public Map toMap(DatabaseType databaseType, int schemaVersion) { throw new UnsupportedOperationException( "EntityNameLookupRecordConverter is read-only and does not support toMap operation"); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java index bddcb35953..d2662daa5e 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java @@ -35,7 +35,25 @@ public class ModelEntity implements Converter { public static final String ID_COLUMN = "id"; - public static final List ALL_COLUMNS = + private static final List ALL_COLUMNS = + List.of( + "id", + "catalog_id", + "parent_id", + "type_code", + "name", + "entity_version", + "sub_type_code", + "create_timestamp", + "drop_timestamp", + "purge_timestamp", + "to_purge_timestamp", + "last_update_timestamp", + "properties", + "internal_properties", + "grant_records_version"); + + private static final List ALL_COLUMNS_V2 = List.of( "id", "catalog_id", @@ -54,6 +72,14 @@ public class ModelEntity implements Converter { "grant_records_version", "location_without_scheme"); + public static List getAllColumnNames(int schemaVersion) { + if (schemaVersion < 2) { + return ALL_COLUMNS; + } else { + return ALL_COLUMNS_V2; + } + } + public static final List ENTITY_LOOKUP_COLUMNS = List.of("id", "catalog_id", "parent_id", "type_code", "name", "sub_type_code"); @@ -176,6 +202,14 @@ public static Builder builder() { @Override public PolarisBaseEntity fromResultSet(ResultSet r) throws SQLException { + // location_without_scheme column was added in schema version 2 + String locationWithoutScheme = null; + try { + locationWithoutScheme = r.getString("location_without_scheme"); + } catch (SQLException e) { + // Column does not exist, handle gracefully + } + var modelEntity = ModelEntity.builder() .catalogId(r.getObject("catalog_id", Long.class)) @@ -195,14 +229,14 @@ public PolarisBaseEntity fromResultSet(ResultSet r) throws SQLException { // JSONB: use getString(), not getObject(). .internalProperties(r.getString("internal_properties")) .grantRecordsVersion(r.getObject("grant_records_version", Integer.class)) - .locationWithoutScheme(r.getString("location_without_scheme")) + .locationWithoutScheme(locationWithoutScheme) .build(); return toEntity(modelEntity); } @Override - public Map toMap(DatabaseType databaseType) { + public Map toMap(DatabaseType databaseType, int schemaVersion) { Map map = new LinkedHashMap<>(); map.put("id", this.getId()); map.put("catalog_id", this.getCatalogId()); @@ -224,7 +258,9 @@ public Map toMap(DatabaseType databaseType) { map.put("internal_properties", this.getInternalProperties()); } map.put("grant_records_version", this.getGrantRecordsVersion()); - map.put("location_without_scheme", this.getLocationWithoutScheme()); + if (schemaVersion >= 2) { + map.put("location_without_scheme", this.getLocationWithoutScheme()); + } return map; } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java index 92d01c3e81..841aada9fd 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java @@ -92,7 +92,7 @@ default PolarisEvent fromResultSet(ResultSet rs) throws SQLException { } @Override - default Map toMap(DatabaseType databaseType) { + default Map toMap(DatabaseType databaseType, int schemaVersion) { Map map = new LinkedHashMap<>(); map.put("catalog_id", getCatalogId()); map.put("event_id", getEventId()); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java index b853dfd241..cd0015f27f 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java @@ -93,7 +93,7 @@ public PolarisGrantRecord fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType) { + public Map toMap(DatabaseType databaseType, int schemaVersion) { Map map = new LinkedHashMap<>(); map.put("securable_catalog_id", this.securableCatalogId); map.put("securable_id", this.securableId); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java index ab4faa5d44..0e41c5faf3 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java @@ -168,7 +168,7 @@ public PolarisPolicyMappingRecord fromResultSet(ResultSet rs) throws SQLExceptio } @Override - public Map toMap(DatabaseType databaseType) { + public Map toMap(DatabaseType databaseType, int schemaVersion) { Map map = new LinkedHashMap<>(); map.put("target_catalog_id", targetCatalogId); map.put("target_id", targetId); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java index 9013d66421..b6c5ccc69b 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java @@ -90,7 +90,7 @@ public PolarisPrincipalSecrets fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType) { + public Map toMap(DatabaseType databaseType, int schemaVersion) { Map map = new LinkedHashMap<>(); map.put("principal_id", this.principalId); map.put("principal_client_id", this.principalClientId); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java index dc8db05b36..625aa574cc 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java @@ -50,7 +50,7 @@ public SchemaVersion fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType) { + public Map toMap(DatabaseType databaseType, int schemaVersion) { return Map.of("version_value", this.value); } } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index dfc8a84a01..a300b58e80 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -34,16 +34,17 @@ import org.h2.jdbcx.JdbcConnectionPool; import org.mockito.Mockito; -public class AtomicMetastoreManagerWithJdbcBasePersistenceImplTest +public abstract class AtomicMetastoreManagerWithJdbcBasePersistenceImplTest extends BasePolarisMetaStoreManagerTest { public static DataSource createH2DataSource() { return JdbcConnectionPool.create("jdbc:h2:file:./build/test_data/polaris/db", "sa", ""); } + public abstract int schemaVersion(); + @Override protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { - int schemaVersion = 2; PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); DatasourceOperations datasourceOperations; try { @@ -52,7 +53,8 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); InputStream scriptStream = classLoader.getResourceAsStream( - String.format("%s/schema-v%s.sql", DatabaseType.H2.getDisplayName(), schemaVersion)); + String.format( + "%s/schema-v%s.sql", DatabaseType.H2.getDisplayName(), schemaVersion())); datasourceOperations.executeScript(scriptStream); } catch (SQLException e) { throw new RuntimeException( @@ -69,7 +71,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { RANDOM_SECRETS, Mockito.mock(), realmContext.getRealmIdentifier(), - schemaVersion); + schemaVersion()); AtomicOperationMetaStoreManager metaStoreManager = new AtomicOperationMetaStoreManager(clock, diagServices); PolarisCallContext callCtx = new PolarisCallContext(realmContext, basePersistence); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV1SchemaTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV1SchemaTest.java new file mode 100644 index 0000000000..2d50b27015 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV1SchemaTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.relational.jdbc; + +public class AtomicMetastoreManagerWithJdbcBasePersistenceImplV1SchemaTest + extends AtomicMetastoreManagerWithJdbcBasePersistenceImplTest { + @Override + public int schemaVersion() { + return 1; + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV2SchemaTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV2SchemaTest.java new file mode 100644 index 0000000000..8ccafb9618 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV2SchemaTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.relational.jdbc; + +public class AtomicMetastoreManagerWithJdbcBasePersistenceImplV2SchemaTest + extends AtomicMetastoreManagerWithJdbcBasePersistenceImplTest { + @Override + public int schemaVersion() { + return 2; + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV3SchemaTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV3SchemaTest.java new file mode 100644 index 0000000000..bb81da4d02 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV3SchemaTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.relational.jdbc; + +public class AtomicMetastoreManagerWithJdbcBasePersistenceImplV3SchemaTest + extends AtomicMetastoreManagerWithJdbcBasePersistenceImplTest { + @Override + public int schemaVersion() { + return 3; + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java index f9d5c69428..607cc507e0 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java @@ -119,7 +119,7 @@ void executeBatchUpdate_success() throws Exception { .additionalProperties("") .build(); queryParams.add( - modelEvent.toMap(datasourceOperations.getDatabaseType()).values().stream().toList()); + modelEvent.toMap(datasourceOperations.getDatabaseType(), 2).values().stream().toList()); } when(mockConnection.prepareStatement(any())).thenReturn(mockPreparedStatement); when(mockPreparedStatement.executeBatch()).thenReturn(new int[] {100}); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java index 6df78eff57..68cf73ca40 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java @@ -49,7 +49,7 @@ void testGenerateSelectQuery_withMaQueryGeneratorpWhereClause() { assertEquals( expectedQuery, QueryGenerator.generateSelectQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, whereClause) + ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, whereClause) .sql()); } @@ -74,7 +74,8 @@ void testGenerateSelectQueryWithEntityIds_singleId() { String expectedQuery = "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, location_without_scheme FROM POLARIS_SCHEMA.ENTITIES WHERE (catalog_id, id) IN ((?, ?)) AND realm_id = ?"; assertEquals( - expectedQuery, QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, entityIds).sql()); + expectedQuery, + QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, 2, entityIds).sql()); } @Test @@ -84,7 +85,8 @@ void testGenerateSelectQueryWithEntityIds_multipleIds() { String expectedQuery = "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, location_without_scheme FROM POLARIS_SCHEMA.ENTITIES WHERE (catalog_id, id) IN ((?, ?), (?, ?)) AND realm_id = ?"; assertEquals( - expectedQuery, QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, entityIds).sql()); + expectedQuery, + QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, 2, entityIds).sql()); } @Test @@ -92,7 +94,7 @@ void testGenerateSelectQueryWithEntityIds_emptyList() { List entityIds = Collections.emptyList(); assertThrows( IllegalArgumentException.class, - () -> QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, entityIds).sql()); + () -> QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, 2, entityIds).sql()); } @Test @@ -103,9 +105,9 @@ void testGenerateInsertQuery_nonNullFields() { assertEquals( expectedQuery, QueryGenerator.generateInsertQuery( - ModelEntity.ALL_COLUMNS, + ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, - entity.toMap(DatabaseType.H2).values().stream().toList(), + entity.toMap(DatabaseType.H2, 2).values().stream().toList(), REALM_ID) .sql()); } @@ -120,9 +122,9 @@ void testGenerateUpdateQuery_nonNullFields() { assertEquals( expectedQuery, QueryGenerator.generateUpdateQuery( - ModelEntity.ALL_COLUMNS, + ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, - entity.toMap(DatabaseType.H2).values().stream().toList(), + entity.toMap(DatabaseType.H2, 2).values().stream().toList(), whereClause) .sql()); } @@ -137,9 +139,9 @@ void testGenerateUpdateQuery_partialNonNullFields() { assertEquals( expectedQuery, QueryGenerator.generateUpdateQuery( - ModelEntity.ALL_COLUMNS, + ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, - entity.toMap(DatabaseType.H2).values().stream().toList(), + entity.toMap(DatabaseType.H2, 2).values().stream().toList(), whereClause) .sql()); } @@ -152,7 +154,7 @@ void testGenerateDeleteQuery_withMapWhereClause() { assertEquals( expectedQuery, QueryGenerator.generateDeleteQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, whereClause) + ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, whereClause) .sql()); } @@ -162,20 +164,21 @@ void testGenerateDeleteQuery_withStringWhereClause() { assertEquals( expectedQuery, QueryGenerator.generateDeleteQuery( - ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, Map.of("name", "oldName")) + ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, Map.of("name", "oldName")) .sql()); } @Test void testGenerateDeleteQuery_byObject() { ModelEntity entityToDelete = ModelEntity.builder().name("test").entityVersion(1).build(); - Map objMap = entityToDelete.toMap(DatabaseType.H2); + Map objMap = entityToDelete.toMap(DatabaseType.H2, 2); objMap.put("realm_id", REALM_ID); String expectedQuery = "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE id = ? AND catalog_id = ? AND parent_id = ? AND type_code = ? AND name = ? AND entity_version = ? AND sub_type_code = ? AND create_timestamp = ? AND drop_timestamp = ? AND purge_timestamp = ? AND to_purge_timestamp = ? AND last_update_timestamp = ? AND properties = ? AND internal_properties = ? AND grant_records_version = ? AND location_without_scheme = ? AND realm_id = ?"; assertEquals( expectedQuery, - QueryGenerator.generateDeleteQuery(ModelEntity.ALL_COLUMNS, ModelEntity.TABLE_NAME, objMap) + QueryGenerator.generateDeleteQuery( + ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, objMap) .sql()); } @@ -225,9 +228,9 @@ void testGenerateOverlapQuery() { + " POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ?" + " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR" + " location_without_scheme = ? OR location_without_scheme LIKE ?)", - QueryGenerator.generateOverlapQuery("realmId", -123, "s3://bucket/tmp/location/").sql()); + QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://bucket/tmp/location/").sql()); Assertions.assertThatCollection( - QueryGenerator.generateOverlapQuery("realmId", -123, "s3://bucket/tmp/location/") + QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://bucket/tmp/location/") .parameters()) .containsExactly( "realmId", @@ -245,9 +248,9 @@ void testGenerateOverlapQuery() { + " properties, internal_properties, grant_records_version, location_without_scheme FROM" + " POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ? OR location_without_scheme = ?" + " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme LIKE ?)", - QueryGenerator.generateOverlapQuery("realmId", -123, "/tmp/location/").sql()); + QueryGenerator.generateOverlapQuery("realmId", 2, -123, "/tmp/location/").sql()); Assertions.assertThatCollection( - QueryGenerator.generateOverlapQuery("realmId", -123, "/tmp/location/").parameters()) + QueryGenerator.generateOverlapQuery("realmId", 2, -123, "/tmp/location/").parameters()) .containsExactly( "realmId", -123L, "/", "//", "///", "///tmp/", "///tmp/location/", "///tmp/location/%"); @@ -257,9 +260,9 @@ void testGenerateOverlapQuery() { + " properties, internal_properties, grant_records_version, location_without_scheme" + " FROM POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ?" + " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme LIKE ?)", - QueryGenerator.generateOverlapQuery("realmId", -123, "s3://バケツ/\"loc.ation\"/").sql()); + QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://バケツ/\"loc.ation\"/").sql()); Assertions.assertThatCollection( - QueryGenerator.generateOverlapQuery("realmId", -123, "s3://バケツ/\"loc.ation\"/") + QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://バケツ/\"loc.ation\"/") .parameters()) .containsExactly( "realmId", -123L, "/", "//", "//バケツ/", "//バケツ/\"loc.ation\"/", "//バケツ/\"loc.ation\"/%"); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java index fa78de0885..cd5d2f2ffa 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java @@ -125,7 +125,7 @@ public void testToMapWithH2DatabaseType() { .build(); // Act - Map resultMap = modelEvent.toMap(DatabaseType.H2); + Map resultMap = modelEvent.toMap(DatabaseType.H2, 2); // Assert assertEquals(TEST_CATALOG_ID, resultMap.get(CATALOG_ID)); @@ -156,7 +156,7 @@ public void testToMapWithPostgresType() { .build(); // Act - Map resultMap = modelEvent.toMap(DatabaseType.POSTGRES); + Map resultMap = modelEvent.toMap(DatabaseType.POSTGRES, 2); // Assert assertEquals(TEST_CATALOG_ID, resultMap.get(CATALOG_ID)); diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 336b6b5d03..43a0c55815 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -68,7 +68,7 @@ public abstract class BasePolarisMetaStoreManagerTest { private PolarisTestMetaStoreManager polarisTestMetaStoreManager; @BeforeEach - public void setupPolariMetaStoreManager() { + public void setupPolarisMetaStoreManager() { this.polarisTestMetaStoreManager = createPolarisTestMetaStoreManager(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java index 50d0746910..b4bce6f205 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java @@ -302,4 +302,36 @@ public ProductionReadinessCheck checkInsecureStorageSettings( ? ProductionReadinessCheck.OK : ProductionReadinessCheck.of(errors.toArray(new Error[0])); } + + @Produces + public ProductionReadinessCheck checkOverlappingSiblingCheckSettings( + FeaturesConfiguration featureConfiguration) { + var optimizedSiblingCheck = FeatureConfiguration.OPTIMIZED_SIBLING_CHECK; + var errors = new ArrayList(); + if (Boolean.parseBoolean(featureConfiguration.defaults().get(optimizedSiblingCheck.key()))) { + errors.add( + Error.ofSevere( + "This setting is not recommended for production environments as it may lead to incorrect behavior, due to missing data for location_without_scheme column in case of migrating from older Polaris versions." + + optimizedSiblingCheck.description(), + format("polaris.features.\"%s\"", optimizedSiblingCheck.key()))); + } + + featureConfiguration + .realmOverrides() + .forEach( + (realmId, overrides) -> { + if (Boolean.parseBoolean(overrides.overrides().get(optimizedSiblingCheck.key()))) { + errors.add( + Error.ofSevere( + "This setting is not recommended for production environments as it may lead to incorrect behavior, due to missing data for location_without_scheme column in case of migrating from older Polaris versions. " + + optimizedSiblingCheck.description(), + format( + "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"", + realmId, optimizedSiblingCheck.key()))); + } + }); + return errors.isEmpty() + ? ProductionReadinessCheck.OK + : ProductionReadinessCheck.of(errors.toArray(new Error[0])); + } } From b42cc1620c180e8aabd7550b783a83aaee985497 Mon Sep 17 00:00:00 2001 From: Prashant Date: Tue, 30 Sep 2025 19:04:18 -0700 Subject: [PATCH 2/6] Address review feedback --- .../jdbc/JdbcBasePersistenceImpl.java | 55 ++++----- .../relational/jdbc/models/Converter.java | 2 +- .../EntityNameLookupRecordConverter.java | 2 +- .../relational/jdbc/models/ModelEntity.java | 36 ++++-- .../relational/jdbc/models/ModelEvent.java | 2 +- .../jdbc/models/ModelGrantRecord.java | 2 +- .../jdbc/models/ModelPolicyMappingRecord.java | 2 +- .../ModelPrincipalAuthenticationData.java | 2 +- .../relational/jdbc/models/SchemaVersion.java | 2 +- .../src/main/resources/h2/schema-v0.sql | 106 ++++++++++++++++++ ...anagerWithJdbcBasePersistenceImplTest.java | 4 +- ...thJdbcBasePersistenceImplV0SchemaTest.java | 28 +++++ .../jdbc/DatasourceOperationsTest.java | 4 +- .../relational/jdbc/QueryGeneratorTest.java | 11 +- .../jdbc/models/ModelEventTest.java | 4 +- 15 files changed, 198 insertions(+), 64 deletions(-) create mode 100644 persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql create mode 100644 persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV0SchemaTest.java diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 9b6cba3492..0d90fe2774 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -174,15 +174,11 @@ private void persistEntity( Connection connection, QueryAction queryAction) throws SQLException { - ModelEntity modelEntity = ModelEntity.fromEntity(entity); + ModelEntity modelEntity = ModelEntity.fromEntity(entity, schemaVersion); if (originalEntity == null) { try { List values = - modelEntity - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) - .values() - .stream() - .toList(); + modelEntity.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(); queryAction.apply( connection, QueryGenerator.generateInsertQuery( @@ -224,11 +220,7 @@ private void persistEntity( realmId); try { List values = - modelEntity - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) - .values() - .stream() - .toList(); + modelEntity.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(); int rowsUpdated = queryAction.apply( connection, @@ -255,11 +247,7 @@ public void writeToGrantRecords( ModelGrantRecord modelGrantRecord = ModelGrantRecord.fromGrantRecord(grantRec); try { List values = - modelGrantRecord - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) - .values() - .stream() - .toList(); + modelGrantRecord.toMap(datasourceOperations.getDatabaseType()).values().stream().toList(); datasourceOperations.executeUpdate( QueryGenerator.generateInsertQuery( ModelGrantRecord.ALL_COLUMNS, ModelGrantRecord.TABLE_NAME, values, realmId)); @@ -282,7 +270,7 @@ public void writeEvents(@Nonnull List events) { ModelEvent.ALL_COLUMNS, ModelEvent.TABLE_NAME, ModelEvent.fromEvent(events.getFirst()) - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .toMap(datasourceOperations.getDatabaseType()) .values() .stream() .toList(), @@ -300,7 +288,7 @@ public void writeEvents(@Nonnull List events) { ModelEvent.ALL_COLUMNS, ModelEvent.TABLE_NAME, ModelEvent.fromEvent(event) - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .toMap(datasourceOperations.getDatabaseType()) .values() .stream() .toList(), @@ -328,7 +316,7 @@ public void writeEvents(@Nonnull List events) { @Override public void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - ModelEntity modelEntity = ModelEntity.fromEntity(entity); + ModelEntity modelEntity = ModelEntity.fromEntity(entity, schemaVersion); Map params = Map.of( "id", @@ -353,7 +341,7 @@ public void deleteFromGrantRecords( ModelGrantRecord modelGrantRecord = ModelGrantRecord.fromGrantRecord(grantRec); try { Map whereClause = - modelGrantRecord.toMap(datasourceOperations.getDatabaseType(), schemaVersion); + modelGrantRecord.toMap(datasourceOperations.getDatabaseType()); whereClause.put("realm_id", realmId); datasourceOperations.executeUpdate( QueryGenerator.generateDeleteQuery( @@ -450,7 +438,7 @@ public PolarisBaseEntity lookupEntityByName( @Nullable private PolarisBaseEntity getPolarisBaseEntity(QueryGenerator.PreparedQuery query) { try { - var results = datasourceOperations.executeSelect(query, new ModelEntity()); + var results = datasourceOperations.executeSelect(query, new ModelEntity(schemaVersion)); if (results.isEmpty()) { return null; } else if (results.size() > 1) { @@ -475,7 +463,7 @@ public List lookupEntities( PreparedQuery query = QueryGenerator.generateSelectQueryWithEntityIds(realmId, schemaVersion, entityIds); try { - return datasourceOperations.executeSelect(query, new ModelEntity()); + return datasourceOperations.executeSelect(query, new ModelEntity(schemaVersion)); } catch (SQLException e) { throw new RuntimeException( String.format("Failed to retrieve polaris entities due to %s", e.getMessage()), e); @@ -491,7 +479,7 @@ public List lookupEntityVersions( .collect( Collectors.toMap( entry -> new PolarisEntityId(entry.getCatalogId(), entry.getId()), - ModelEntity::fromEntity)); + entry -> ModelEntity.fromEntity(entry, schemaVersion))); return entityIds.stream() .map( entityId -> { @@ -603,7 +591,7 @@ public Page loadEntities( AtomicReference> results = new AtomicReference<>(); datasourceOperations.executeSelectOverStream( query, - new ModelEntity(), + new ModelEntity(schemaVersion), stream -> { var data = stream.filter(entityFilter); results.set(Page.mapped(pageToken, data, transformer, EntityIdToken::fromEntity)); @@ -739,7 +727,7 @@ public boolean hasChildren( datasourceOperations.executeSelect( QueryGenerator.generateSelectQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params), - new ModelEntity()); + new ModelEntity(schemaVersion)); return results != null && !results.isEmpty(); } catch (SQLException e) { throw new RuntimeException( @@ -785,7 +773,7 @@ Optional> hasOverlappingSiblings( QueryGenerator.generateOverlapQuery( realmId, schemaVersion, entity.getCatalogId(), entity.getBaseLocation()); try { - var results = datasourceOperations.executeSelect(query, new ModelEntity()); + var results = datasourceOperations.executeSelect(query, new ModelEntity(schemaVersion)); if (!results.isEmpty()) { StorageLocation entityLocation = StorageLocation.of(entity.getBaseLocation()); for (PolarisBaseEntity result : results) { @@ -859,10 +847,7 @@ public PolarisPrincipalSecrets generateNewPrincipalSecrets( // write new principal secrets try { List values = - lookupPrincipalSecrets - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) - .values() - .stream() + lookupPrincipalSecrets.toMap(datasourceOperations.getDatabaseType()).values().stream() .toList(); datasourceOperations.executeUpdate( QueryGenerator.generateInsertQuery( @@ -902,7 +887,7 @@ public PolarisPrincipalSecrets storePrincipalSecrets( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, modelPrincipalAuthenticationData - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .toMap(datasourceOperations.getDatabaseType()) .values() .stream() .toList(), @@ -963,7 +948,7 @@ public PolarisPrincipalSecrets rotatePrincipalSecrets( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, modelPrincipalAuthenticationData - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .toMap(datasourceOperations.getDatabaseType()) .values() .stream() .toList(), @@ -1017,7 +1002,7 @@ public void writeToPolicyMappingRecords( ModelPolicyMappingRecord.fromPolicyMappingRecord(record); List values = modelPolicyMappingRecord - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .toMap(datasourceOperations.getDatabaseType()) .values() .stream() .toList(); @@ -1081,7 +1066,7 @@ private boolean handleInheritablePolicy( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, modelPolicyMappingRecord - .toMap(datasourceOperations.getDatabaseType(), schemaVersion) + .toMap(datasourceOperations.getDatabaseType()) .values() .stream() .toList(), @@ -1100,7 +1085,7 @@ public void deleteFromPolicyMappingRecords( var modelPolicyMappingRecord = ModelPolicyMappingRecord.fromPolicyMappingRecord(record); try { Map objectMap = - modelPolicyMappingRecord.toMap(datasourceOperations.getDatabaseType(), schemaVersion); + modelPolicyMappingRecord.toMap(datasourceOperations.getDatabaseType()); objectMap.put("realm_id", realmId); datasourceOperations.executeUpdate( QueryGenerator.generateDeleteQuery( diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java index fa29a243cb..fcf5d1dfcc 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java @@ -38,7 +38,7 @@ public interface Converter { * Convert a model into a Map with keys as snake case names, and values as values of member of * model obj. */ - Map toMap(DatabaseType databaseType, int schemaVersion); + Map toMap(DatabaseType databaseType); default PGobject toJsonbPGobject(String props) { try { diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java index ca34986ec9..d4bd7775bb 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityNameLookupRecordConverter.java @@ -39,7 +39,7 @@ public EntityNameLookupRecord fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType, int schemaVersion) { + public Map toMap(DatabaseType databaseType) { throw new UnsupportedOperationException( "EntityNameLookupRecordConverter is read-only and does not support toMap operation"); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java index d2662daa5e..c5c3c4261b 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java @@ -132,6 +132,16 @@ public static List getAllColumnNames(int schemaVersion) { // location for the entity but without a scheme, when applicable private String locationWithoutScheme; + // schema version of the entity + // NOTE: this field is not stored in the database, but is used to handle schema changes + private int schemaVersion; + + public ModelEntity(int schemaVersion) { + this.schemaVersion = schemaVersion; + } + + public ModelEntity() {} + public long getId() { return id; } @@ -196,19 +206,16 @@ public String getLocationWithoutScheme() { return locationWithoutScheme; } + public int getSchemaVersion() { + return schemaVersion; + } + public static Builder builder() { return new Builder(); } @Override public PolarisBaseEntity fromResultSet(ResultSet r) throws SQLException { - // location_without_scheme column was added in schema version 2 - String locationWithoutScheme = null; - try { - locationWithoutScheme = r.getString("location_without_scheme"); - } catch (SQLException e) { - // Column does not exist, handle gracefully - } var modelEntity = ModelEntity.builder() @@ -229,14 +236,14 @@ public PolarisBaseEntity fromResultSet(ResultSet r) throws SQLException { // JSONB: use getString(), not getObject(). .internalProperties(r.getString("internal_properties")) .grantRecordsVersion(r.getObject("grant_records_version", Integer.class)) - .locationWithoutScheme(locationWithoutScheme) + .locationWithoutScheme(this.schemaVersion >= 2 ? locationWithoutScheme : null) .build(); return toEntity(modelEntity); } @Override - public Map toMap(DatabaseType databaseType, int schemaVersion) { + public Map toMap(DatabaseType databaseType) { Map map = new LinkedHashMap<>(); map.put("id", this.getId()); map.put("catalog_id", this.getCatalogId()); @@ -258,7 +265,7 @@ public Map toMap(DatabaseType databaseType, int schemaVersion) { map.put("internal_properties", this.getInternalProperties()); } map.put("grant_records_version", this.getGrantRecordsVersion()); - if (schemaVersion >= 2) { + if (this.getSchemaVersion() >= 2) { map.put("location_without_scheme", this.getLocationWithoutScheme()); } return map; @@ -351,12 +358,17 @@ public Builder locationWithoutScheme(String location) { return this; } + public Builder schemaVersion(int schemaVersion) { + entity.schemaVersion = schemaVersion; + return this; + } + public ModelEntity build() { return entity; } } - public static ModelEntity fromEntity(PolarisBaseEntity entity) { + public static ModelEntity fromEntity(PolarisBaseEntity entity, int schemaVersion) { var builder = ModelEntity.builder() .catalogId(entity.getCatalogId()) @@ -391,6 +403,8 @@ public static ModelEntity fromEntity(PolarisBaseEntity entity) { .withoutScheme()); } + builder.schemaVersion(schemaVersion); + return builder.build(); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java index 841aada9fd..92d01c3e81 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java @@ -92,7 +92,7 @@ default PolarisEvent fromResultSet(ResultSet rs) throws SQLException { } @Override - default Map toMap(DatabaseType databaseType, int schemaVersion) { + default Map toMap(DatabaseType databaseType) { Map map = new LinkedHashMap<>(); map.put("catalog_id", getCatalogId()); map.put("event_id", getEventId()); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java index cd0015f27f..b853dfd241 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelGrantRecord.java @@ -93,7 +93,7 @@ public PolarisGrantRecord fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType, int schemaVersion) { + public Map toMap(DatabaseType databaseType) { Map map = new LinkedHashMap<>(); map.put("securable_catalog_id", this.securableCatalogId); map.put("securable_id", this.securableId); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java index 0e41c5faf3..ab4faa5d44 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java @@ -168,7 +168,7 @@ public PolarisPolicyMappingRecord fromResultSet(ResultSet rs) throws SQLExceptio } @Override - public Map toMap(DatabaseType databaseType, int schemaVersion) { + public Map toMap(DatabaseType databaseType) { Map map = new LinkedHashMap<>(); map.put("target_catalog_id", targetCatalogId); map.put("target_id", targetId); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java index b6c5ccc69b..9013d66421 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPrincipalAuthenticationData.java @@ -90,7 +90,7 @@ public PolarisPrincipalSecrets fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType, int schemaVersion) { + public Map toMap(DatabaseType databaseType) { Map map = new LinkedHashMap<>(); map.put("principal_id", this.principalId); map.put("principal_client_id", this.principalClientId); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java index 625aa574cc..dc8db05b36 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/SchemaVersion.java @@ -50,7 +50,7 @@ public SchemaVersion fromResultSet(ResultSet rs) throws SQLException { } @Override - public Map toMap(DatabaseType databaseType, int schemaVersion) { + public Map toMap(DatabaseType databaseType) { return Map.of("version_value", this.value); } } diff --git a/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql b/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql new file mode 100644 index 0000000000..5c0e1c9c7a --- /dev/null +++ b/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql @@ -0,0 +1,106 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + + +-- TEST ONLY, TO SIMULATE already having v1 in place +CREATE TABLE IF NOT EXISTS entities ( + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + id BIGINT NOT NULL, + parent_id BIGINT NOT NULL, + name TEXT NOT NULL, + entity_version INT NOT NULL, + type_code INT NOT NULL, + sub_type_code INT NOT NULL, + create_timestamp BIGINT NOT NULL, + drop_timestamp BIGINT NOT NULL, + purge_timestamp BIGINT NOT NULL, + to_purge_timestamp BIGINT NOT NULL, + last_update_timestamp BIGINT NOT NULL, + properties TEXT NOT NULL DEFAULT '{}', + internal_properties TEXT NOT NULL DEFAULT '{}', + grant_records_version INT NOT NULL, + PRIMARY KEY (realm_id, id), +CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) +); + +-- TODO: create indexes based on all query pattern. +CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); + +COMMENT ON TABLE entities IS 'all the entities'; + +COMMENT ON COLUMN entities.catalog_id IS 'catalog id'; +COMMENT ON COLUMN entities.id IS 'entity id'; +COMMENT ON COLUMN entities.parent_id IS 'entity id of parent'; +COMMENT ON COLUMN entities.name IS 'entity name'; +COMMENT ON COLUMN entities.entity_version IS 'version of the entity'; +COMMENT ON COLUMN entities.type_code IS 'type code'; +COMMENT ON COLUMN entities.sub_type_code IS 'sub type of entity'; +COMMENT ON COLUMN entities.create_timestamp IS 'creation time of entity'; +COMMENT ON COLUMN entities.drop_timestamp IS 'time of drop of entity'; +COMMENT ON COLUMN entities.purge_timestamp IS 'time to start purging entity'; +COMMENT ON COLUMN entities.last_update_timestamp IS 'last time the entity is touched'; +COMMENT ON COLUMN entities.properties IS 'entities properties json'; +COMMENT ON COLUMN entities.internal_properties IS 'entities internal properties json'; +COMMENT ON COLUMN entities.grant_records_version IS 'the version of grant records change on the entity'; + +DROP TABLE IF EXISTS grant_records; +CREATE TABLE IF NOT EXISTS grant_records ( + realm_id TEXT NOT NULL, + securable_catalog_id BIGINT NOT NULL, + securable_id BIGINT NOT NULL, + grantee_catalog_id BIGINT NOT NULL, + grantee_id BIGINT NOT NULL, + privilege_code INTEGER, + PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code) +); + +COMMENT ON TABLE grant_records IS 'grant records for entities'; +COMMENT ON COLUMN grant_records.securable_catalog_id IS 'catalog id of the securable'; +COMMENT ON COLUMN grant_records.securable_id IS 'entity id of the securable'; +COMMENT ON COLUMN grant_records.grantee_catalog_id IS 'catalog id of the grantee'; +COMMENT ON COLUMN grant_records.grantee_id IS 'id of the grantee'; +COMMENT ON COLUMN grant_records.privilege_code IS 'privilege code'; + +DROP TABLE IF EXISTS principal_authentication_data; +CREATE TABLE IF NOT EXISTS principal_authentication_data ( + realm_id TEXT NOT NULL, + principal_id BIGINT NOT NULL, + principal_client_id VARCHAR(255) NOT NULL, + main_secret_hash VARCHAR(255) NOT NULL, + secondary_secret_hash VARCHAR(255) NOT NULL, + secret_salt VARCHAR(255) NOT NULL, + PRIMARY KEY (realm_id, principal_client_id) +); + +COMMENT ON TABLE principal_authentication_data IS 'authentication data for client'; + +DROP TABLE IF EXISTS policy_mapping_record; +CREATE TABLE IF NOT EXISTS policy_mapping_record ( + realm_id TEXT NOT NULL, + target_catalog_id BIGINT NOT NULL, + target_id BIGINT NOT NULL, + policy_type_code INTEGER NOT NULL, + policy_catalog_id BIGINT NOT NULL, + policy_id BIGINT NOT NULL, + parameters TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id) +); + +CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index a300b58e80..c7a6d4132d 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -37,8 +37,8 @@ public abstract class AtomicMetastoreManagerWithJdbcBasePersistenceImplTest extends BasePolarisMetaStoreManagerTest { - public static DataSource createH2DataSource() { - return JdbcConnectionPool.create("jdbc:h2:file:./build/test_data/polaris/db", "sa", ""); + public DataSource createH2DataSource() { + return JdbcConnectionPool.create(String.format("jdbc:h2:file:./build/test_data/polaris/db_%s", schemaVersion()), "sa", ""); } public abstract int schemaVersion(); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV0SchemaTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV0SchemaTest.java new file mode 100644 index 0000000000..3650d8b7ac --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplV0SchemaTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.persistence.relational.jdbc; + +public class AtomicMetastoreManagerWithJdbcBasePersistenceImplV0SchemaTest + extends AtomicMetastoreManagerWithJdbcBasePersistenceImplTest { + @Override + public int schemaVersion() { + return 0; + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java index 607cc507e0..6063ad3d87 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java @@ -119,7 +119,7 @@ void executeBatchUpdate_success() throws Exception { .additionalProperties("") .build(); queryParams.add( - modelEvent.toMap(datasourceOperations.getDatabaseType(), 2).values().stream().toList()); + modelEvent.toMap(datasourceOperations.getDatabaseType()).values().stream().toList()); } when(mockConnection.prepareStatement(any())).thenReturn(mockPreparedStatement); when(mockPreparedStatement.executeBatch()).thenReturn(new int[] {100}); @@ -141,7 +141,7 @@ void testExecuteSelect_exception() throws Exception { when(mockPreparedStatement.executeQuery()).thenThrow(new SQLException("demo", "42P07")); assertThrows( - SQLException.class, () -> datasourceOperations.executeSelect(query, new ModelEntity())); + SQLException.class, () -> datasourceOperations.executeSelect(query, new ModelEntity(1))); } @Test diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java index 68cf73ca40..d31b077889 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java @@ -107,7 +107,7 @@ void testGenerateInsertQuery_nonNullFields() { QueryGenerator.generateInsertQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, - entity.toMap(DatabaseType.H2, 2).values().stream().toList(), + entity.toMap(DatabaseType.H2).values().stream().toList(), REALM_ID) .sql()); } @@ -124,7 +124,7 @@ void testGenerateUpdateQuery_nonNullFields() { QueryGenerator.generateUpdateQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, - entity.toMap(DatabaseType.H2, 2).values().stream().toList(), + entity.toMap(DatabaseType.H2).values().stream().toList(), whereClause) .sql()); } @@ -141,7 +141,7 @@ void testGenerateUpdateQuery_partialNonNullFields() { QueryGenerator.generateUpdateQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, - entity.toMap(DatabaseType.H2, 2).values().stream().toList(), + entity.toMap(DatabaseType.H2).values().stream().toList(), whereClause) .sql()); } @@ -170,8 +170,9 @@ void testGenerateDeleteQuery_withStringWhereClause() { @Test void testGenerateDeleteQuery_byObject() { - ModelEntity entityToDelete = ModelEntity.builder().name("test").entityVersion(1).build(); - Map objMap = entityToDelete.toMap(DatabaseType.H2, 2); + ModelEntity entityToDelete = + ModelEntity.builder().name("test").entityVersion(1).schemaVersion(2).build(); + Map objMap = entityToDelete.toMap(DatabaseType.H2); objMap.put("realm_id", REALM_ID); String expectedQuery = "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE id = ? AND catalog_id = ? AND parent_id = ? AND type_code = ? AND name = ? AND entity_version = ? AND sub_type_code = ? AND create_timestamp = ? AND drop_timestamp = ? AND purge_timestamp = ? AND to_purge_timestamp = ? AND last_update_timestamp = ? AND properties = ? AND internal_properties = ? AND grant_records_version = ? AND location_without_scheme = ? AND realm_id = ?"; diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java index cd5d2f2ffa..fa78de0885 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java @@ -125,7 +125,7 @@ public void testToMapWithH2DatabaseType() { .build(); // Act - Map resultMap = modelEvent.toMap(DatabaseType.H2, 2); + Map resultMap = modelEvent.toMap(DatabaseType.H2); // Assert assertEquals(TEST_CATALOG_ID, resultMap.get(CATALOG_ID)); @@ -156,7 +156,7 @@ public void testToMapWithPostgresType() { .build(); // Act - Map resultMap = modelEvent.toMap(DatabaseType.POSTGRES, 2); + Map resultMap = modelEvent.toMap(DatabaseType.POSTGRES); // Assert assertEquals(TEST_CATALOG_ID, resultMap.get(CATALOG_ID)); From 98be0ae3c537ab87e79a1f99b42d6c1755dff712 Mon Sep 17 00:00:00 2001 From: Prashant Date: Tue, 30 Sep 2025 19:29:54 -0700 Subject: [PATCH 3/6] fix typo --- .../persistence/relational/jdbc/models/ModelEntity.java | 2 +- .../relational-jdbc/src/main/resources/h2/schema-v0.sql | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java index c5c3c4261b..d2061b01fd 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java @@ -236,7 +236,7 @@ public PolarisBaseEntity fromResultSet(ResultSet r) throws SQLException { // JSONB: use getString(), not getObject(). .internalProperties(r.getString("internal_properties")) .grantRecordsVersion(r.getObject("grant_records_version", Integer.class)) - .locationWithoutScheme(this.schemaVersion >= 2 ? locationWithoutScheme : null) + .locationWithoutScheme(this.schemaVersion >= 2 ? r.getString("location_without_scheme") : null) .build(); return toEntity(modelEntity); diff --git a/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql b/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql index 5c0e1c9c7a..61aec4bed5 100644 --- a/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql +++ b/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql @@ -19,6 +19,9 @@ -- TEST ONLY, TO SIMULATE already having v1 in place +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET SCHEMA POLARIS_SCHEMA; + CREATE TABLE IF NOT EXISTS entities ( realm_id TEXT NOT NULL, catalog_id BIGINT NOT NULL, From 64106cdeca522c2bb7e5574de125cbd2ecde7ee4 Mon Sep 17 00:00:00 2001 From: Prashant Date: Wed, 1 Oct 2025 04:51:39 -0700 Subject: [PATCH 4/6] spotless; ; --- .../persistence/relational/jdbc/models/ModelEntity.java | 3 ++- .../AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java index d2061b01fd..90893c8baf 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java @@ -236,7 +236,8 @@ public PolarisBaseEntity fromResultSet(ResultSet r) throws SQLException { // JSONB: use getString(), not getObject(). .internalProperties(r.getString("internal_properties")) .grantRecordsVersion(r.getObject("grant_records_version", Integer.class)) - .locationWithoutScheme(this.schemaVersion >= 2 ? r.getString("location_without_scheme") : null) + .locationWithoutScheme( + this.schemaVersion >= 2 ? r.getString("location_without_scheme") : null) .build(); return toEntity(modelEntity); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index c7a6d4132d..dedce84198 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -38,7 +38,8 @@ public abstract class AtomicMetastoreManagerWithJdbcBasePersistenceImplTest extends BasePolarisMetaStoreManagerTest { public DataSource createH2DataSource() { - return JdbcConnectionPool.create(String.format("jdbc:h2:file:./build/test_data/polaris/db_%s", schemaVersion()), "sa", ""); + return JdbcConnectionPool.create( + String.format("jdbc:h2:file:./build/test_data/polaris/db_%s", schemaVersion()), "sa", ""); } public abstract int schemaVersion(); From 4d3733ba2eb83b6e05e51d1928ff74e8465c4f3b Mon Sep 17 00:00:00 2001 From: Prashant Date: Thu, 2 Oct 2025 16:20:13 -0700 Subject: [PATCH 5/6] Address review feedback --- .../src/test/resources/h2/schema-v0.sql | 109 ++++++++++++++++++ .../core/config/FeatureConfiguration.java | 10 ++ .../config/ProductionReadinessChecks.java | 13 ++- 3 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 persistence/relational-jdbc/src/test/resources/h2/schema-v0.sql diff --git a/persistence/relational-jdbc/src/test/resources/h2/schema-v0.sql b/persistence/relational-jdbc/src/test/resources/h2/schema-v0.sql new file mode 100644 index 0000000000..0fc24f365c --- /dev/null +++ b/persistence/relational-jdbc/src/test/resources/h2/schema-v0.sql @@ -0,0 +1,109 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + + +-- TO SIMULATE the schema in Polaris 1.0.x, which doesn't have the version table, Polaris interpret its schema version as 0, even the schema file named "schema-v1.sql". +CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; +SET SCHEMA POLARIS_SCHEMA; + +CREATE TABLE IF NOT EXISTS entities ( + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + id BIGINT NOT NULL, + parent_id BIGINT NOT NULL, + name TEXT NOT NULL, + entity_version INT NOT NULL, + type_code INT NOT NULL, + sub_type_code INT NOT NULL, + create_timestamp BIGINT NOT NULL, + drop_timestamp BIGINT NOT NULL, + purge_timestamp BIGINT NOT NULL, + to_purge_timestamp BIGINT NOT NULL, + last_update_timestamp BIGINT NOT NULL, + properties TEXT NOT NULL DEFAULT '{}', + internal_properties TEXT NOT NULL DEFAULT '{}', + grant_records_version INT NOT NULL, + PRIMARY KEY (realm_id, id), +CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) +); + +-- TODO: create indexes based on all query pattern. +CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); + +COMMENT ON TABLE entities IS 'all the entities'; + +COMMENT ON COLUMN entities.catalog_id IS 'catalog id'; +COMMENT ON COLUMN entities.id IS 'entity id'; +COMMENT ON COLUMN entities.parent_id IS 'entity id of parent'; +COMMENT ON COLUMN entities.name IS 'entity name'; +COMMENT ON COLUMN entities.entity_version IS 'version of the entity'; +COMMENT ON COLUMN entities.type_code IS 'type code'; +COMMENT ON COLUMN entities.sub_type_code IS 'sub type of entity'; +COMMENT ON COLUMN entities.create_timestamp IS 'creation time of entity'; +COMMENT ON COLUMN entities.drop_timestamp IS 'time of drop of entity'; +COMMENT ON COLUMN entities.purge_timestamp IS 'time to start purging entity'; +COMMENT ON COLUMN entities.last_update_timestamp IS 'last time the entity is touched'; +COMMENT ON COLUMN entities.properties IS 'entities properties json'; +COMMENT ON COLUMN entities.internal_properties IS 'entities internal properties json'; +COMMENT ON COLUMN entities.grant_records_version IS 'the version of grant records change on the entity'; + +DROP TABLE IF EXISTS grant_records; +CREATE TABLE IF NOT EXISTS grant_records ( + realm_id TEXT NOT NULL, + securable_catalog_id BIGINT NOT NULL, + securable_id BIGINT NOT NULL, + grantee_catalog_id BIGINT NOT NULL, + grantee_id BIGINT NOT NULL, + privilege_code INTEGER, + PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code) +); + +COMMENT ON TABLE grant_records IS 'grant records for entities'; +COMMENT ON COLUMN grant_records.securable_catalog_id IS 'catalog id of the securable'; +COMMENT ON COLUMN grant_records.securable_id IS 'entity id of the securable'; +COMMENT ON COLUMN grant_records.grantee_catalog_id IS 'catalog id of the grantee'; +COMMENT ON COLUMN grant_records.grantee_id IS 'id of the grantee'; +COMMENT ON COLUMN grant_records.privilege_code IS 'privilege code'; + +DROP TABLE IF EXISTS principal_authentication_data; +CREATE TABLE IF NOT EXISTS principal_authentication_data ( + realm_id TEXT NOT NULL, + principal_id BIGINT NOT NULL, + principal_client_id VARCHAR(255) NOT NULL, + main_secret_hash VARCHAR(255) NOT NULL, + secondary_secret_hash VARCHAR(255) NOT NULL, + secret_salt VARCHAR(255) NOT NULL, + PRIMARY KEY (realm_id, principal_client_id) +); + +COMMENT ON TABLE principal_authentication_data IS 'authentication data for client'; + +DROP TABLE IF EXISTS policy_mapping_record; +CREATE TABLE IF NOT EXISTS policy_mapping_record ( + realm_id TEXT NOT NULL, + target_catalog_id BIGINT NOT NULL, + target_id BIGINT NOT NULL, + policy_type_code INTEGER NOT NULL, + policy_catalog_id BIGINT NOT NULL, + policy_id BIGINT NOT NULL, + parameters TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id) +); + +CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 8af3e18210..2b95ddd67c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -353,6 +353,16 @@ public static void enforceFeatureEnabledOrThrow( .defaultValue(true) .buildFeatureConfiguration(); + public static final FeatureConfiguration ALLOW_OPTIMIZED_SIBLING_CHECK = + PolarisConfiguration.builder() + .key("ALLOW_OPTIMIZED_SIBLING_CHECK") + .description( + "When set to true, Polaris will permit enabling the feature OPTIMIZED_SIBLING_CHECK " + + "for catalogs, this is done to prevent accidental enabling the feature in cases such as schema migrations, without backfill and hence leading to potential data integrity issues.\n" + + "This will be removed in 2.0.0 when polaris ships with the necessary migrations to backfill the index.") + .defaultValue(false) + .buildFeatureConfiguration(); + public static final FeatureConfiguration OPTIMIZED_SIBLING_CHECK = PolarisConfiguration.builder() .key("OPTIMIZED_SIBLING_CHECK") diff --git a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java index b4bce6f205..1c9cedd4ca 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java @@ -307,12 +307,13 @@ public ProductionReadinessCheck checkInsecureStorageSettings( public ProductionReadinessCheck checkOverlappingSiblingCheckSettings( FeaturesConfiguration featureConfiguration) { var optimizedSiblingCheck = FeatureConfiguration.OPTIMIZED_SIBLING_CHECK; + var allowOverlap = FeatureConfiguration.ALLOW_OPTIMIZED_SIBLING_CHECK; var errors = new ArrayList(); - if (Boolean.parseBoolean(featureConfiguration.defaults().get(optimizedSiblingCheck.key()))) { + if (Boolean.parseBoolean(featureConfiguration.defaults().get(optimizedSiblingCheck.key())) + && !Boolean.parseBoolean(featureConfiguration.defaults().get(allowOverlap.key()))) { errors.add( Error.ofSevere( - "This setting is not recommended for production environments as it may lead to incorrect behavior, due to missing data for location_without_scheme column in case of migrating from older Polaris versions." - + optimizedSiblingCheck.description(), + "This setting should be used with care and only enabled in new realms. Enabling it in previously used realms and may lead to incorrect behavior, due to missing data for location_without_scheme column. Set the ALLOW_OPTIMIZED_SIBLING_CHECK flag to acknowledge this warning and enable Polaris to start.", format("polaris.features.\"%s\"", optimizedSiblingCheck.key()))); } @@ -320,11 +321,11 @@ public ProductionReadinessCheck checkOverlappingSiblingCheckSettings( .realmOverrides() .forEach( (realmId, overrides) -> { - if (Boolean.parseBoolean(overrides.overrides().get(optimizedSiblingCheck.key()))) { + if (Boolean.parseBoolean(overrides.overrides().get(optimizedSiblingCheck.key())) + && !Boolean.parseBoolean(overrides.overrides().get(allowOverlap.key()))) { errors.add( Error.ofSevere( - "This setting is not recommended for production environments as it may lead to incorrect behavior, due to missing data for location_without_scheme column in case of migrating from older Polaris versions. " - + optimizedSiblingCheck.description(), + "This setting should be used with care and only enabled in new realms. Enabling it in previously used realms and may lead to incorrect behavior, due to missing data for location_without_scheme column. Set the ALLOW_OPTIMIZED_SIBLING_CHECK flag to acknowledge this warning and enable Polaris to start.", format( "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"", realmId, optimizedSiblingCheck.key()))); From 24fb4ef017bb6b96485ed6d33457188d243c6129 Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 3 Oct 2025 00:47:32 -0700 Subject: [PATCH 6/6] remove redundant file --- .../src/main/resources/h2/schema-v0.sql | 109 ------------------ 1 file changed, 109 deletions(-) delete mode 100644 persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql diff --git a/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql b/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql deleted file mode 100644 index 61aec4bed5..0000000000 --- a/persistence/relational-jdbc/src/main/resources/h2/schema-v0.sql +++ /dev/null @@ -1,109 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file-- --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"). You may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. --- - - --- TEST ONLY, TO SIMULATE already having v1 in place -CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; -SET SCHEMA POLARIS_SCHEMA; - -CREATE TABLE IF NOT EXISTS entities ( - realm_id TEXT NOT NULL, - catalog_id BIGINT NOT NULL, - id BIGINT NOT NULL, - parent_id BIGINT NOT NULL, - name TEXT NOT NULL, - entity_version INT NOT NULL, - type_code INT NOT NULL, - sub_type_code INT NOT NULL, - create_timestamp BIGINT NOT NULL, - drop_timestamp BIGINT NOT NULL, - purge_timestamp BIGINT NOT NULL, - to_purge_timestamp BIGINT NOT NULL, - last_update_timestamp BIGINT NOT NULL, - properties TEXT NOT NULL DEFAULT '{}', - internal_properties TEXT NOT NULL DEFAULT '{}', - grant_records_version INT NOT NULL, - PRIMARY KEY (realm_id, id), -CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) -); - --- TODO: create indexes based on all query pattern. -CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); - -COMMENT ON TABLE entities IS 'all the entities'; - -COMMENT ON COLUMN entities.catalog_id IS 'catalog id'; -COMMENT ON COLUMN entities.id IS 'entity id'; -COMMENT ON COLUMN entities.parent_id IS 'entity id of parent'; -COMMENT ON COLUMN entities.name IS 'entity name'; -COMMENT ON COLUMN entities.entity_version IS 'version of the entity'; -COMMENT ON COLUMN entities.type_code IS 'type code'; -COMMENT ON COLUMN entities.sub_type_code IS 'sub type of entity'; -COMMENT ON COLUMN entities.create_timestamp IS 'creation time of entity'; -COMMENT ON COLUMN entities.drop_timestamp IS 'time of drop of entity'; -COMMENT ON COLUMN entities.purge_timestamp IS 'time to start purging entity'; -COMMENT ON COLUMN entities.last_update_timestamp IS 'last time the entity is touched'; -COMMENT ON COLUMN entities.properties IS 'entities properties json'; -COMMENT ON COLUMN entities.internal_properties IS 'entities internal properties json'; -COMMENT ON COLUMN entities.grant_records_version IS 'the version of grant records change on the entity'; - -DROP TABLE IF EXISTS grant_records; -CREATE TABLE IF NOT EXISTS grant_records ( - realm_id TEXT NOT NULL, - securable_catalog_id BIGINT NOT NULL, - securable_id BIGINT NOT NULL, - grantee_catalog_id BIGINT NOT NULL, - grantee_id BIGINT NOT NULL, - privilege_code INTEGER, - PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code) -); - -COMMENT ON TABLE grant_records IS 'grant records for entities'; -COMMENT ON COLUMN grant_records.securable_catalog_id IS 'catalog id of the securable'; -COMMENT ON COLUMN grant_records.securable_id IS 'entity id of the securable'; -COMMENT ON COLUMN grant_records.grantee_catalog_id IS 'catalog id of the grantee'; -COMMENT ON COLUMN grant_records.grantee_id IS 'id of the grantee'; -COMMENT ON COLUMN grant_records.privilege_code IS 'privilege code'; - -DROP TABLE IF EXISTS principal_authentication_data; -CREATE TABLE IF NOT EXISTS principal_authentication_data ( - realm_id TEXT NOT NULL, - principal_id BIGINT NOT NULL, - principal_client_id VARCHAR(255) NOT NULL, - main_secret_hash VARCHAR(255) NOT NULL, - secondary_secret_hash VARCHAR(255) NOT NULL, - secret_salt VARCHAR(255) NOT NULL, - PRIMARY KEY (realm_id, principal_client_id) -); - -COMMENT ON TABLE principal_authentication_data IS 'authentication data for client'; - -DROP TABLE IF EXISTS policy_mapping_record; -CREATE TABLE IF NOT EXISTS policy_mapping_record ( - realm_id TEXT NOT NULL, - target_catalog_id BIGINT NOT NULL, - target_id BIGINT NOT NULL, - policy_type_code INTEGER NOT NULL, - policy_catalog_id BIGINT NOT NULL, - policy_id BIGINT NOT NULL, - parameters TEXT NOT NULL DEFAULT '{}', - PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id) -); - -CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id);