diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index c53f1140c0..492f61668d 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -28,8 +28,8 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; /** @@ -60,7 +60,7 @@ protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics } @Override - protected PolarisMetaStoreSession createMetaStoreSession( + protected TransactionalPersistence createMetaStoreSession( @Nonnull PolarisEclipseLinkStore store, @Nonnull RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet, diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 7950075c94..4954de2ce9 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -39,20 +39,20 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.exceptions.AlreadyExistsException; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.BaseMetaStoreManager; import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; import org.apache.polaris.core.persistence.RetryOnConcurrencyException; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -68,7 +68,7 @@ * EclipseLink implementation of a Polaris metadata store supporting persisting and retrieving all * Polaris metadata from/to the configured database systems. */ -public class PolarisEclipseLinkMetaStoreSessionImpl implements PolarisMetaStoreSession { +public class PolarisEclipseLinkMetaStoreSessionImpl extends TransactionalPersistence { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisEclipseLinkMetaStoreSessionImpl.class); @@ -271,14 +271,6 @@ public void writeToEntitiesActive( this.store.writeToEntitiesActive(localSession.get(), entity); } - /** {@inheritDoc} */ - @Override - public void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // write it - this.store.writeToEntitiesDropped(localSession.get(), entity); - } - /** {@inheritDoc} */ @Override public void writeToEntitiesChangeTracking( @@ -312,14 +304,6 @@ public void deleteFromEntitiesActive( this.store.deleteFromEntitiesActive(localSession.get(), new PolarisEntitiesActiveKey(entity)); } - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // delete it - this.store.deleteFromEntitiesDropped(localSession.get(), entity.getCatalogId(), entity.getId()); - } - /** * {@inheritDoc} * @@ -371,14 +355,6 @@ public void deleteAll(@Nonnull PolarisCallContext callCtx) { .toList(); } - /** {@inheritDoc} */ - @Override - public int lookupEntityVersion( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId) { - ModelEntity model = this.store.lookupEntity(localSession.get(), catalogId, entityId); - return model == null ? 0 : model.getEntityVersion(); - } - /** {@inheritDoc} */ @Override public @Nonnull List lookupEntityVersions( @@ -404,7 +380,7 @@ public int lookupEntityVersion( /** {@inheritDoc} */ @Override @Nullable - public PolarisEntityActiveRecord lookupEntityActive( + public EntityNameLookupRecord lookupEntityActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey) { // lookup the active entity slice return ModelEntityActive.toEntityActive( @@ -414,7 +390,7 @@ public PolarisEntityActiveRecord lookupEntityActive( /** {@inheritDoc} */ @Override @Nonnull - public List lookupEntityActiveBatch( + public List lookupEntityActiveBatch( @Nonnull PolarisCallContext callCtx, @Nonnull List entityActiveKeys) { // now build a list to quickly verify that nothing has changed @@ -425,23 +401,23 @@ public List lookupEntityActiveBatch( /** {@inheritDoc} */ @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); + return listEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType, @Nonnull Predicate entityFilter) { // full range scan under the parent for that type - return listActiveEntities( + return listEntities( callCtx, catalogId, parentId, @@ -449,7 +425,7 @@ public List lookupEntityActiveBatch( Integer.MAX_VALUE, entityFilter, entity -> - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( entity.getCatalogId(), entity.getId(), entity.getParentId(), @@ -459,7 +435,7 @@ public List lookupEntityActiveBatch( } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @@ -674,7 +650,7 @@ PolarisStorageIntegration createStorageIntegration( PolarisStorageIntegration loadPolarisStorageIntegration( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { PolarisStorageConfigurationInfo storageConfig = - PolarisMetaStoreManagerImpl.readStorageConfiguration(callCtx, entity); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, entity); return storageIntegrationProvider.getStorageIntegrationForConfig(storageConfig); } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 8216bef91f..bfc83ae373 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -27,9 +27,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -38,7 +38,6 @@ import org.apache.polaris.jpa.models.ModelEntity; import org.apache.polaris.jpa.models.ModelEntityActive; import org.apache.polaris.jpa.models.ModelEntityChangeTracking; -import org.apache.polaris.jpa.models.ModelEntityDropped; import org.apache.polaris.jpa.models.ModelGrantRecord; import org.apache.polaris.jpa.models.ModelPrincipalSecrets; import org.slf4j.Logger; @@ -100,18 +99,7 @@ void writeToEntitiesActive(EntityManager session, PolarisBaseEntity entity) { ModelEntityActive model = lookupEntityActive(session, new PolarisEntitiesActiveKey(entity)); if (model == null) { - session.persist(ModelEntityActive.fromEntityActive(new PolarisEntityActiveRecord(entity))); - } - } - - void writeToEntitiesDropped(EntityManager session, PolarisBaseEntity entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entityDropped = - lookupEntityDropped(session, entity.getCatalogId(), entity.getId()); - if (entityDropped == null) { - session.persist(ModelEntityDropped.fromEntity(entity)); + session.persist(ModelEntityActive.fromEntityActive(new EntityNameLookupRecord(entity))); } } @@ -158,16 +146,6 @@ void deleteFromEntitiesActive(EntityManager session, PolarisEntitiesActiveKey ke session.remove(entity); } - void deleteFromEntitiesDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entity = lookupEntityDropped(session, catalogId, entityId); - diagnosticServices.check(entity != null, "dropped_entity_not_found"); - - session.remove(entity); - } - void deleteFromEntitiesChangeTracking(EntityManager session, PolarisEntityCore entity) { diagnosticServices.check(session != null, "session_is_null"); checkInitialized(); @@ -216,7 +194,6 @@ void deleteAll(EntityManager session) { session.createQuery("DELETE from ModelEntity").executeUpdate(); session.createQuery("DELETE from ModelEntityActive").executeUpdate(); - session.createQuery("DELETE from ModelEntityDropped").executeUpdate(); session.createQuery("DELETE from ModelEntityChangeTracking").executeUpdate(); session.createQuery("DELETE from ModelGrantRecord").executeUpdate(); session.createQuery("DELETE from ModelPrincipalSecrets").executeUpdate(); @@ -319,21 +296,6 @@ List lookupFullEntitiesActive( return query.getResultList(); } - ModelEntityDropped lookupEntityDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelEntityDropped m where m.catalogId=:catalogId and m.id=:id", - ModelEntityDropped.class) - .setParameter("catalogId", catalogId) - .setParameter("id", entityId) - .getResultStream() - .findFirst() - .orElse(null); - } - ModelEntityChangeTracking lookupEntityChangeTracking( EntityManager session, long catalogId, long entityId) { diagnosticServices.check(session != null, "session_is_null"); diff --git a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java index cd9b085902..9415696dae 100644 --- a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java +++ b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java @@ -41,8 +41,8 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; import org.apache.polaris.jpa.models.ModelPrincipalSecrets; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; diff --git a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java b/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java index 2dab162ee1..56770d1be2 100644 --- a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java +++ b/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java @@ -21,7 +21,7 @@ import jakarta.persistence.Entity; import jakarta.persistence.Id; import jakarta.persistence.Table; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -128,7 +128,7 @@ public ModelEntityActive build() { } } - public static ModelEntityActive fromEntityActive(PolarisEntityActiveRecord record) { + public static ModelEntityActive fromEntityActive(EntityNameLookupRecord record) { return ModelEntityActive.builder() .catalogId(record.getCatalogId()) .id(record.getId()) @@ -139,12 +139,12 @@ public static ModelEntityActive fromEntityActive(PolarisEntityActiveRecord recor .build(); } - public static PolarisEntityActiveRecord toEntityActive(ModelEntityActive model) { + public static EntityNameLookupRecord toEntityActive(ModelEntityActive model) { if (model == null) { return null; } - return new PolarisEntityActiveRecord( + return new EntityNameLookupRecord( model.catalogId, model.id, model.parentId, model.name, model.typeCode, model.subTypeCode); } } diff --git a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java b/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java deleted file mode 100644 index 97986e7928..0000000000 --- a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.jpa.models; - -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import jakarta.persistence.Version; -import org.apache.polaris.core.entity.PolarisBaseEntity; - -/** - * EntityDropped model representing some attributes of a Polaris Entity. This is used to exchange - * entity information with ENTITIES_DROPPED table - */ -@Entity -@Table(name = "ENTITIES_DROPPED") -public class ModelEntityDropped { - // the id of the catalog associated to that entity. NULL_ID if this entity is top-level like - // a catalog - @Id private long catalogId; - - // the id of the entity which was resolved - private long id; - - // the id of the parent of this entity, use 0 for a top-level entity whose parent is the account - @Id private long parentId; - - // the type of the entity when it was resolved - @Id private int typeCode; - - // the name that this entity had when it was resolved - @Id private String name; - - // the type of the entity when it was resolved - @Id private int subTypeCode; - - // when this entity was dropped. Null if was never dropped - @Id private long dropTimestamp; - - // when should we start purging this entity - private long toPurgeTimestamp; - - // Used for Optimistic Locking to handle concurrent reads and updates - @Version private long version; - - public long getCatalogId() { - return catalogId; - } - - public long getId() { - return id; - } - - public long getParentId() { - return parentId; - } - - public int getTypeCode() { - return typeCode; - } - - public String getName() { - return name; - } - - public int getSubTypeCode() { - return subTypeCode; - } - - public long getDropTimestamp() { - return dropTimestamp; - } - - public long getToPurgeTimestamp() { - return toPurgeTimestamp; - } - - public static Builder builder() { - return new Builder(); - } - - public static final class Builder { - private final ModelEntityDropped entity; - - private Builder() { - entity = new ModelEntityDropped(); - } - - public Builder catalogId(long catalogId) { - entity.catalogId = catalogId; - return this; - } - - public Builder id(long id) { - entity.id = id; - return this; - } - - public Builder parentId(long parentId) { - entity.parentId = parentId; - return this; - } - - public Builder typeCode(int typeCode) { - entity.typeCode = typeCode; - return this; - } - - public Builder name(String name) { - entity.name = name; - return this; - } - - public Builder subTypeCode(int subTypeCode) { - entity.subTypeCode = subTypeCode; - return this; - } - - public Builder dropTimestamp(long dropTimestamp) { - entity.dropTimestamp = dropTimestamp; - return this; - } - - public Builder toPurgeTimestamp(long toPurgeTimestamp) { - entity.toPurgeTimestamp = toPurgeTimestamp; - return this; - } - - public ModelEntityDropped build() { - return entity; - } - } - - public static ModelEntityDropped fromEntity(PolarisBaseEntity entity) { - if (entity == null) return null; - - return ModelEntityDropped.builder() - .catalogId(entity.getCatalogId()) - .id(entity.getId()) - .parentId(entity.getParentId()) - .typeCode(entity.getTypeCode()) - .name(entity.getName()) - .subTypeCode(entity.getSubTypeCode()) - .dropTimestamp(entity.getDropTimestamp()) - .toPurgeTimestamp(entity.getToPurgeTimestamp()) - .build(); - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java index abe598714f..c584dde55c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java @@ -21,7 +21,7 @@ import jakarta.annotation.Nonnull; import java.time.Clock; import java.time.ZoneId; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; /** * The Call context is allocated each time a new REST request is processed. It contains instances of @@ -30,7 +30,7 @@ public class PolarisCallContext { // meta store which is used to persist Polaris entity metadata - private final PolarisMetaStoreSession metaStore; + private final TransactionalPersistence metaStore; // diag services private final PolarisDiagnostics diagServices; @@ -40,7 +40,7 @@ public class PolarisCallContext { private final Clock clock; public PolarisCallContext( - @Nonnull PolarisMetaStoreSession metaStore, + @Nonnull TransactionalPersistence metaStore, @Nonnull PolarisDiagnostics diagServices, @Nonnull PolarisConfigurationStore configurationStore, @Nonnull Clock clock) { @@ -51,7 +51,7 @@ public PolarisCallContext( } public PolarisCallContext( - @Nonnull PolarisMetaStoreSession metaStore, @Nonnull PolarisDiagnostics diagServices) { + @Nonnull TransactionalPersistence metaStore, @Nonnull PolarisDiagnostics diagServices) { this.metaStore = metaStore; this.diagServices = diagServices; this.configurationStore = new PolarisConfigurationStore() {}; @@ -66,7 +66,7 @@ public static PolarisCallContext copyOf(PolarisCallContext original) { original.getClock()); } - public PolarisMetaStoreSession getMetaStore() { + public TransactionalPersistence getMetaStore() { return metaStore; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityActiveRecord.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/EntityNameLookupRecord.java similarity index 92% rename from polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityActiveRecord.java rename to polaris-core/src/main/java/org/apache/polaris/core/entity/EntityNameLookupRecord.java index 87682f4cb2..6d150a50f0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityActiveRecord.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/EntityNameLookupRecord.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; -public class PolarisEntityActiveRecord { +public class EntityNameLookupRecord { // entity catalog id private final long catalogId; @@ -74,7 +74,7 @@ public PolarisEntitySubType getSubType() { } @JsonCreator - public PolarisEntityActiveRecord( + public EntityNameLookupRecord( @JsonProperty("catalogId") long catalogId, @JsonProperty("id") long id, @JsonProperty("parentId") long parentId, @@ -90,7 +90,7 @@ public PolarisEntityActiveRecord( } /** Constructor to create the object with provided entity */ - public PolarisEntityActiveRecord(PolarisBaseEntity entity) { + public EntityNameLookupRecord(PolarisBaseEntity entity) { this.catalogId = entity.getCatalogId(); this.id = entity.getId(); this.parentId = entity.getParentId(); @@ -102,8 +102,8 @@ public PolarisEntityActiveRecord(PolarisBaseEntity entity) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof PolarisEntityActiveRecord)) return false; - PolarisEntityActiveRecord that = (PolarisEntityActiveRecord) o; + if (!(o instanceof EntityNameLookupRecord)) return false; + EntityNameLookupRecord that = (EntityNameLookupRecord) o; return catalogId == that.catalogId && id == that.id && parentId == that.parentId diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java index 3ded968af2..c68d1c974d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java @@ -172,7 +172,7 @@ public static List toCoreList(List path) { .orElse(null); } - public static List toNameAndIdList(List entities) { + public static List toNameAndIdList(List entities) { return Optional.ofNullable(entities) .map( list -> diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BaseMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BaseMetaStoreManager.java new file mode 100644 index 0000000000..c27ee1b377 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BaseMetaStoreManager.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import java.util.Map; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; + +public abstract class BaseMetaStoreManager implements PolarisMetaStoreManager { + public static PolarisStorageConfigurationInfo extractStorageConfiguration( + @Nonnull PolarisCallContext callCtx, PolarisBaseEntity reloadedEntity) { + Map propMap = + PolarisObjectMapperUtil.deserializeProperties( + callCtx, reloadedEntity.getInternalProperties()); + String storageConfigInfoStr = + propMap.get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + + callCtx + .getDiagServices() + .check( + storageConfigInfoStr != null, + "missing_storage_configuration_info", + "catalogId={}, entityId={}", + reloadedEntity.getCatalogId(), + reloadedEntity.getId()); + return PolarisStorageConfigurationInfo.deserialize( + callCtx.getDiagServices(), storageConfigInfoStr); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java new file mode 100644 index 0000000000..31c2d502d8 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; + +/** + * Interface to the Polaris persistence backend, with which to persist and retrieve all the data + * defining the internal data model for Polaris, and which defines the basis for the RBAC model + * provided by Polaris. + * + *

Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * The returned id must be fully unique within a realm and never reused once generated, whether or + * not anything ends up committing an entity with the generated id. + * + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write this entity to the persistence backend. If successful, the write must be durable and + * visible to any other reader. + * + *

TODO: Either standardize the expected system of exceptions to throw for various concurrency + * errors (entity not found when originalEntity != null, entity changed from originalEntity, etc) + * or push down the return status enums from PolarisMetaStoreManager into this layer and document + * accordingly. + * + * @param callCtx call context + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity + */ + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); + + /** + * Atomically write a batch of entities to the persistence backend conditional on *every* member + * of originalEntities matching the existing persistent state. After this commit, *every* member + * of entities must be committed durably. + * + *

TODO: Push down the multi-entity commit from PolarisMetaStoreManagerImpl to use this instead + * of running single writeEntity actions within a transaction. + * + * @param callCtx call context + * @param entities entities to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntities original states of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity; must contain exactly as many elements as + * {@code entities} where each item corresponds to the element of {@code entities} in the same + * index as this list. + */ + void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List entities, + @Nullable List originalEntities); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete this entity from the meta store. + * + * @param callCtx call context + * @param entity entity to delete + */ + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List grantsOnGrantee, + @Nonnull List grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables within the realm defined by the + * contents of the {@code callCtx} + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities) and its + * entityId. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId entity id + * @return null if the entity was not found, else the retrieved entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities like + * CATALOG, PRINCIPAL and PRINCIPAL_ROLE. Note that by convention, a catalog itself has + * NULL_ID for its catalogId since the catalog is not "nested" under itself or any other + * catalog. + * @param parentId id of the parent + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name); + + /** + * Looks up just the entity's subType and id given it catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + default EntityNameLookupRecord lookupEntityIdAndSubTypeByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + PolarisBaseEntity baseEntity = lookupEntityByName(callCtx, catalogId, parentId, typeCode, name); + if (baseEntity == null) { + return null; + } + return new EntityNameLookupRecord(baseEntity); + } + + /** + * Lookup a set of entities given their catalog id/entity id unique identifier + * + * @param callCtx call context + * @param entityIds list of entity ids + * @return list of polaris base entities, parallel to the input list of ids. An entity in the list + * will be null if the corresponding entity could not be found. + */ + @Nonnull + List lookupEntities( + @Nonnull PolarisCallContext callCtx, List entityIds); + + /** + * Get change tracking versions for all specified entity ids. + * + * @param callCtx call context + * @param entityIds list of entity id + * @return list parallel to the input list of entity versions. If an entity cannot be found, the + * corresponding element in the list will be null + */ + @Nonnull + List lookupEntityVersions( + @Nonnull PolarisCallContext callCtx, List entityIds); + + /** + * List all entities of the specified type which are child entities of the specified parent + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @return the list of entities for the specified list operation + */ + @Nonnull + List listEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType); + + /** + * List entities where some predicate returns true + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @return the list of entities for which the predicate returns true + */ + @Nonnull + List listEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull Predicate entityFilter); + + /** + * List entities where some predicate returns true and transform the entities with a function + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param limit the max number of items to return + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before + * returning + * @return the list of entities for which the predicate returns true + */ + @Nonnull + List listEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + int limit, + @Nonnull Predicate entityFilter, + @Nonnull Function transformer); + + /** + * Lookup the current entityGrantRecordsVersion for the specified entity. That version is changed + * everytime a grant record is added or removed on a base securable or added to a grantee. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return current grant records version for that entity. + */ + int lookupEntityGrantRecordsVersion( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup the specified grant record from the grant_records table. Return NULL if not found + * + * @param callCtx call context + * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is + * top-level + * @param securableId id of the securable entity + * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level + * @param granteeId id of the grantee entity + * @param privilegeCode code for the privilege we are looking up + * @return the grant record if found, NULL if not found + */ + @Nullable + PolarisGrantRecord lookupGrantRecord( + @Nonnull PolarisCallContext callCtx, + long securableCatalogId, + long securableId, + long granteeCatalogId, + long granteeId, + int privilegeCode); + + /** + * Get all grant records on the specified securable entity. + * + * @param callCtx call context + * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is + * top-level + * @param securableId id of the securable entity + * @return the list of grant records for the specified securable + */ + @Nonnull + List loadAllGrantRecordsOnSecurable( + @Nonnull PolarisCallContext callCtx, long securableCatalogId, long securableId); + + /** + * Get all grant records granted to the specified grantee entity. + * + * @param callCtx call context + * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level + * @param granteeId id of the grantee entity + * @return the list of grant records for the specified grantee + */ + @Nonnull + List loadAllGrantRecordsOnGrantee( + @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId); + + /** + * Check if the specified parent entity has children. + * + *

TODO: Figure out if this is needed vs listEntities with limit. + * + * @param callContext the polaris call context + * @param optionalEntityType if not null, only check for the specified type, else check for all + * types of children entities + * @param catalogId id of the catalog + * @param parentId id of the parent + * @return true if the parent entity has children + */ + boolean hasChildren( + @Nonnull PolarisCallContext callContext, + @Nullable PolarisEntityType optionalEntityType, + long catalogId, + long parentId); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java new file mode 100644 index 0000000000..92a66dbfa6 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface for the necessary "peripheral integration" objects that are logically attached to core + * persistence entities but which typically involve additional separate external integrations + * related to identity/auth, kms/secrets storage, etc. + * + *

Implementations should orchestrate any necessary multi-phase protocols such as leasing an + * external resource before committing a reference to the external resource in the Polaris + * persistence layer, etc. + */ +public interface IntegrationPersistence { + /** + * Allows to retrieve to the secrets of a principal given its unique client id + * + * @param callCtx call context + * @param clientId principal client id + * @return the secrets + */ + @Nullable + PolarisPrincipalSecrets loadPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); + + /** + * generate and store a client id and associated secrets for a newly created principal entity + * + * @param callCtx call context + * @param principalName name of the principal + * @param principalId principal id + */ + @Nonnull + PolarisPrincipalSecrets generateNewPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); + + /** + * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary + * and generate a new main secret + * + * @param callCtx call context + * @param clientId principal client id + * @param principalId principal id + * @param reset true if the principal secrets should be disabled and replaced with a one-time + * password + * @param oldSecretHash the principal secret's old main secret hash + */ + @Nullable + PolarisPrincipalSecrets rotatePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + @Nonnull String clientId, + long principalId, + boolean reset, + @Nonnull String oldSecretHash); + + /** + * When dropping a principal, we also need to drop the secrets of that principal + * + * @param callCtx the call context + * @param clientId principal client id + * @param principalId the id of the principal whose secrets are dropped + */ + void deletePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId); + + /** + * Create an in-memory storage integration + * + * @param callCtx the polaris calllctx + * @param catalogId the catalog id + * @param entityId the entity id + * @param polarisStorageConfigurationInfo the storage configuration information + * @return a storage integration object + */ + @Nullable + PolarisStorageIntegration createStorageIntegration( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long entityId, + PolarisStorageConfigurationInfo polarisStorageConfigurationInfo); + + /** + * Persist a storage integration in the metastore + * + * @param callContext the polaris call context + * @param entity the entity of the object + * @param storageIntegration the storage integration to persist + */ + void persistStorageIntegrationIfNeeded( + @Nonnull PolarisCallContext callContext, + @Nonnull PolarisBaseEntity entity, + @Nullable PolarisStorageIntegration storageIntegration); + + /** + * Load the polaris storage integration for a polaris entity (Catalog,Namespace,Table,View) + * + * @param callContext the polaris call context + * @param entity the polaris entity + * @return a polaris storage integration + */ + @Nullable + + PolarisStorageIntegration loadPolarisStorageIntegration( + @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entity); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 8c7d3bb5a8..0ea992ed38 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -37,6 +37,8 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +55,7 @@ public abstract class LocalPolarisMetaStoreManagerFactory final Map storageCredentialCacheMap = new HashMap<>(); final Map entityCacheMap = new HashMap<>(); final Map backingStoreMap = new HashMap<>(); - final Map> sessionSupplierMap = new HashMap<>(); + final Map> sessionSupplierMap = new HashMap<>(); protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); private static final Logger LOGGER = @@ -68,7 +70,7 @@ protected LocalPolarisMetaStoreManagerFactory(@Nonnull PolarisDiagnostics diagno protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); - protected abstract PolarisMetaStoreSession createMetaStoreSession( + protected abstract TransactionalPersistence createMetaStoreSession( @Nonnull StoreType store, @Nonnull RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet, @@ -118,7 +120,7 @@ public synchronized Map bootstrapRealms( public void purgeRealms(List realms) { for (String realm : realms) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(() -> realm); - PolarisMetaStoreSession session = getOrCreateSessionSupplier(() -> realm).get(); + TransactionalPersistence session = getOrCreateSessionSupplier(() -> realm).get(); PolarisCallContext callContext = new PolarisCallContext(session, diagServices); metaStoreManager.purge(callContext); @@ -142,7 +144,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( } @Override - public synchronized Supplier getOrCreateSessionSupplier( + public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) { initializeForRealm(realmContext, null); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index 1355715786..56338fb571 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.cache.StorageCredentialCache; /** Configuration interface for configuring the {@link PolarisMetaStoreManager}. */ @@ -32,7 +33,7 @@ public interface MetaStoreManagerFactory { PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext); - Supplier getOrCreateSessionSupplier(RealmContext realmContext); + Supplier getOrCreateSessionSupplier(RealmContext realmContext); StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 30c72a26a1..8a364ef65b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -28,10 +28,10 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisGrantManager; import org.apache.polaris.core.auth.PolarisSecretsManager; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -175,7 +175,7 @@ PolarisMetaStoreManager.EntityResult readEntityByName( class ListEntitiesResult extends BaseResult { // null if not success. Else the list of entities being returned - private final List entities; + private final List entities; /** * Constructor for an error @@ -194,7 +194,7 @@ public ListEntitiesResult( * * @param entities list of entities being returned, implies success */ - public ListEntitiesResult(@Nonnull List entities) { + public ListEntitiesResult(@Nonnull List entities) { super(ReturnStatus.SUCCESS); this.entities = entities; } @@ -203,12 +203,12 @@ public ListEntitiesResult(@Nonnull List entities) { private ListEntitiesResult( @JsonProperty("returnStatus") @Nonnull ReturnStatus returnStatus, @JsonProperty("extraInformation") String extraInformation, - @JsonProperty("entities") List entities) { + @JsonProperty("entities") List entities) { super(returnStatus, extraInformation); this.entities = entities; } - public List getEntities() { + public List getEntities() { return entities; } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java deleted file mode 100644 index 124ec1d7a9..0000000000 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ /dev/null @@ -1,527 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.core.persistence; - -import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import java.util.List; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; -import org.apache.polaris.core.entity.PolarisEntityCore; -import org.apache.polaris.core.entity.PolarisEntityId; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.apache.polaris.core.storage.PolarisStorageIntegration; - -/** - * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like - * metadata for Polaris entities and metadata about grants between these entities which is the - * foundation of our role base access control model. - * - *

Note that APIs to the actual persistence store are very basic, often point read or write to - * the underlying data store. The goal is to make it really easy to back this using databases like - * Postgres or simpler KV store. - */ -public interface PolarisMetaStoreSession { - - /** - * Run the specified transaction code (a Supplier lambda type) in a database read/write - * transaction. If the code of the transaction does not throw any exception and returns normally, - * the transaction will be committed, else the transaction will be automatically rolled-back on - * error. The result of the supplier lambda is returned if success, else the error will be - * re-thrown. - * - * @param callCtx call context - * @param transactionCode code of the transaction being executed, a supplier lambda - */ - T runInTransaction(@Nonnull PolarisCallContext callCtx, @Nonnull Supplier transactionCode); - - /** - * Run the specified transaction code (a runnable lambda type) in a database read/write - * transaction. If the code of the transaction does not throw any exception and returns normally, - * the transaction will be committed, else the transaction will be automatically rolled-back on - * error. - * - * @param callCtx call context - * @param transactionCode code of the transaction being executed, a runnable lambda - */ - void runActionInTransaction( - @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); - - /** - * Run the specified transaction code (a Supplier lambda type) in a database read transaction. If - * the code of the transaction does not throw any exception and returns normally, the transaction - * will be committed, else the transaction will be automatically rolled-back on error. The result - * of the supplier lambda is returned if success, else the error will be re-thrown. - * - * @param callCtx call context - * @param transactionCode code of the transaction being executed, a supplier lambda - */ - T runInReadTransaction( - @Nonnull PolarisCallContext callCtx, @Nonnull Supplier transactionCode); - - /** - * Run the specified transaction code (a runnable lambda type) in a database read transaction. If - * the code of the transaction does not throw any exception and returns normally, the transaction - * will be committed, else the transaction will be automatically rolled-back on error. - * - * @param callCtx call context - * @param transactionCode code of the transaction being executed, a runnable lambda - */ - void runActionInReadTransaction( - @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); - - /** - * @param callCtx call context - * @return new unique entity identifier - */ - long generateNewId(@Nonnull PolarisCallContext callCtx); - - /** - * Write the base entity to the entities table. If there is a conflict (existing record with the - * same id), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key - */ - void writeToEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - - /** - * Write the base entity to the entities_active table. If there is a conflict (existing record - * with the same PK), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key - */ - void writeToEntitiesActive( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - - /** - * Write the base entity to the entities_dropped table. If there is a conflict (existing record - * with the same PK), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key - */ - void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - - /** - * Write the base entity to the entities change tracking table. If there is a conflict (existing - * record with the same id), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key - */ - void writeToEntitiesChangeTracking( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - - /** - * Write the specified grantRecord to the grant_records table. If there is a conflict (existing - * record with the same PK), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param grantRec entity record to write, potentially replacing an existing entity record with - * the same key - */ - void writeToGrantRecords( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); - - /** - * Delete the base entity from the entities table. - * - * @param callCtx call context - * @param entity entity record to delete - */ - void deleteFromEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); - - /** - * Delete the base entity from the entities_active table. - * - * @param callCtx call context - * @param entity entity record to delete - */ - void deleteFromEntitiesActive( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); - - /** - * Delete the base entity to the entities_dropped table - * - * @param callCtx call context - * @param entity entity record to delete - */ - void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - - /** - * Delete the base entity from the entities change tracking table - * - * @param callCtx call context - * @param entity entity record to delete - */ - void deleteFromEntitiesChangeTracking( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); - - /** - * Delete the specified grantRecord to the grant_records table. - * - * @param callCtx call context - * @param grantRec entity record to delete. - */ - void deleteFromGrantRecords( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); - - /** - * Delete the all grant records in the grant_records table for the specified entity. This method - * will delete all grant records on that securable entity and also all grants to that grantee - * entity assuming that the entity is a grantee (catalog role, principal role or principal). - * - * @param callCtx call context - * @param entity entity whose grant records to and from should be deleted - * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a - * grantee - * @param grantsOnSecurable all grants on that securable entity - */ - void deleteAllEntityGrantRecords( - @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisEntityCore entity, - @Nonnull List grantsOnGrantee, - @Nonnull List grantsOnSecurable); - - /** - * Delete Polaris entity and grant record metadata from all tables. This is used during metadata - * bootstrap to reset all tables to their original state - * - * @param callCtx call context - */ - void deleteAll(@Nonnull PolarisCallContext callCtx); - - /** - * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its - * unique id. - * - * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return NULL if the entity was not found, else the base entity. - */ - @Nullable - PolarisBaseEntity lookupEntity( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); - - /** - * Lookup a set of entities given their catalog id/entity id unique identifier - * - * @param callCtx call context - * @param entityIds list of entity ids - * @return list of polaris base entities, parallel to the input list of ids. An entity in the list - * will be null if the corresponding entity could not be found. - */ - @Nonnull - List lookupEntities( - @Nonnull PolarisCallContext callCtx, List entityIds); - - /** - * Lookup in the entities_change_tracking table the current version of an entity given its catalog - * id (which can be NULL_ID for top-level entities) and its unique id. Will return 0 if the entity - * does not exist. - * - * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return current version for that entity or 0 if entity was not found. - */ - int lookupEntityVersion(@Nonnull PolarisCallContext callCtx, long catalogId, long entityId); - - /** - * Get change tracking versions for all specified entity ids. - * - * @param callCtx call context - * @param entityIds list of entity id - * @return list parallel to the input list of entity versions. If an entity cannot be found, the - * corresponding element in the list will be null - */ - @Nonnull - List lookupEntityVersions( - @Nonnull PolarisCallContext callCtx, List entityIds); - - /** - * Lookup in the entities_active table to determine if the specified entity exists. Return the - * result of that lookup - * - * @param callCtx call context - * @param entityActiveKey key in the ENTITIES_ACTIVE table - * @return null if the specified entity does not exist or has been dropped. - */ - @Nullable - PolarisEntityActiveRecord lookupEntityActive( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); - - /** - * Lookup in the entities_active table to determine if the specified set of entities exist. Return - * the result, a parallel list of active records. A record in that list will be null if its - * associated lookup failed - * - * @return the list of entities_active records for the specified lookup operation - */ - @Nonnull - List lookupEntityActiveBatch( - @Nonnull PolarisCallContext callCtx, List entityActiveKeys); - - /** - * List all active entities of the specified type which are child entities of the specified parent - * - * @param callCtx call context - * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level - * @param parentId id of the parent, can be the special 0 value representing the root entity - * @param entityType type of entities to list - * @return the list of entities_active records for the specified list operation - */ - @Nonnull - List listActiveEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType); - - /** - * List active entities where some predicate returns true - * - * @param callCtx call context - * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level - * @param parentId id of the parent, can be the special 0 value representing the root entity - * @param entityType type of entities to list - * @param entityFilter the filter to be applied to each entity. Only entities where the predicate - * returns true are returned in the list - * @return the list of entities for which the predicate returns true - */ - @Nonnull - List listActiveEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter); - - /** - * List active entities where some predicate returns true and transform the entities with a - * function - * - * @param callCtx call context - * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level - * @param parentId id of the parent, can be the special 0 value representing the root entity - * @param entityType type of entities to list - * @param limit the max number of items to return - * @param entityFilter the filter to be applied to each entity. Only entities where the predicate - * returns true are returned in the list - * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before - * returning - * @return the list of entities for which the predicate returns true - */ - @Nonnull - List listActiveEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - int limit, - @Nonnull Predicate entityFilter, - @Nonnull Function transformer); - - /** - * Lookup in the entities_change_tracking table the current version of the grant records for this - * entity. That version is changed everytime a grant record is added or removed on a base - * securable or added to a grantee. - * - * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return current grant records version for that entity. - */ - int lookupEntityGrantRecordsVersion( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); - - /** - * Lookup the specified grant record from the grant_records table. Return NULL if not found - * - * @param callCtx call context - * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is - * top-level - * @param securableId id of the securable entity - * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level - * @param granteeId id of the grantee entity - * @param privilegeCode code for the privilege we are looking up - * @return the grant record if found, NULL if not found - */ - @Nullable - PolarisGrantRecord lookupGrantRecord( - @Nonnull PolarisCallContext callCtx, - long securableCatalogId, - long securableId, - long granteeCatalogId, - long granteeId, - int privilegeCode); - - /** - * Get all grant records on the specified securable entity. - * - * @param callCtx call context - * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is - * top-level - * @param securableId id of the securable entity - * @return the list of grant records for the specified securable - */ - @Nonnull - List loadAllGrantRecordsOnSecurable( - @Nonnull PolarisCallContext callCtx, long securableCatalogId, long securableId); - - /** - * Get all grant records granted to the specified grantee entity. - * - * @param callCtx call context - * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level - * @param granteeId id of the grantee entity - * @return the list of grant records for the specified grantee - */ - @Nonnull - List loadAllGrantRecordsOnGrantee( - @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId); - - /** - * Allows to retrieve to the secrets of a principal given its unique client id - * - * @param callCtx call context - * @param clientId principal client id - * @return the secrets - */ - @Nullable - PolarisPrincipalSecrets loadPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); - - /** - * generate and store a client id and associated secrets for a newly created principal entity - * - * @param callCtx call context - * @param principalName name of the principal - * @param principalId principal id - */ - @Nonnull - PolarisPrincipalSecrets generateNewPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); - - /** - * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary - * and generate a new main secret - * - * @param callCtx call context - * @param clientId principal client id - * @param principalId principal id - * @param reset true if the principal secrets should be disabled and replaced with a one-time - * password - * @param oldSecretHash the principal secret's old main secret hash - */ - @Nullable - PolarisPrincipalSecrets rotatePrincipalSecrets( - @Nonnull PolarisCallContext callCtx, - @Nonnull String clientId, - long principalId, - boolean reset, - @Nonnull String oldSecretHash); - - /** - * When dropping a principal, we also need to drop the secrets of that principal - * - * @param callCtx the call context - * @param clientId principal client id - * @param principalId the id of the principal whose secrets are dropped - */ - void deletePrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId); - - /** - * Create an in-memory storage integration - * - * @param callCtx the polaris calllctx - * @param catalogId the catalog id - * @param entityId the entity id - * @param polarisStorageConfigurationInfo the storage configuration information - * @return a storage integration object - */ - @Nullable - PolarisStorageIntegration createStorageIntegration( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long entityId, - PolarisStorageConfigurationInfo polarisStorageConfigurationInfo); - - /** - * Persist a storage integration in the metastore - * - * @param callContext the polaris call context - * @param entity the entity of the object - * @param storageIntegration the storage integration to persist - */ - void persistStorageIntegrationIfNeeded( - @Nonnull PolarisCallContext callContext, - @Nonnull PolarisBaseEntity entity, - @Nullable PolarisStorageIntegration storageIntegration); - - /** - * Load the polaris storage integration for a polaris entity (Catalog,Namespace,Table,View) - * - * @param callContext the polaris call context - * @param entity the polaris entity - * @return a polaris storage integration - */ - @Nullable - - PolarisStorageIntegration loadPolarisStorageIntegration( - @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entity); - - /** - * Check if the specified parent entity has children. - * - * @param callContext the polaris call context - * @param optionalEntityType if not null, only check for the specified type, else check for all - * types of children entities - * @param catalogId id of the catalog - * @param parentId id of the parent, either a namespace or a catalog - * @return true if the parent entity has children - */ - boolean hasChildren( - @Nonnull PolarisCallContext callContext, - @Nullable PolarisEntityType optionalEntityType, - long catalogId, - long parentId); - - /** Rollback the current transaction */ - void rollback(); -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java index a91d2bbab5..89b0c6429f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java @@ -114,9 +114,9 @@ public static Map deserializeProperties( return retProperties; } - static class TaskExecutionState { - final String executor; - final long lastAttemptStartTime; + public static class TaskExecutionState { + public final String executor; + public final long lastAttemptStartTime; final int attemptCount; TaskExecutionState(String executor, long lastAttemptStartTime, int attemptCount) { @@ -146,7 +146,7 @@ public int getAttemptCount() { * @param entity entity * @return TaskExecutionState */ - static @Nullable TaskExecutionState parseTaskState(PolarisBaseEntity entity) { + public static @Nullable TaskExecutionState parseTaskState(PolarisBaseEntity entity) { JsonFactory jfactory = new JsonFactory(); try (JsonParser jParser = jfactory.createParser(entity.getProperties())) { String executorId = null; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisEntityResolver.java similarity index 96% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisEntityResolver.java index 3d795d7359..bb4e39591e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisEntityResolver.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; @@ -26,9 +26,9 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityType; @@ -78,7 +78,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nullable PolarisEntityCore resolvedEntity, @Nullable List otherTopLevelEntities) { @@ -157,7 +157,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath) { this(callCtx, ms, catalogPath, null, null); } @@ -173,7 +173,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, PolarisEntityCore resolvedEntityDto) { this(callCtx, ms, catalogPath, resolvedEntityDto, null); @@ -190,7 +190,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { this(callCtx, ms, catalogPath, new PolarisEntityCore(entity), null); @@ -239,7 +239,7 @@ long getCatalogIdOrNull() { */ private boolean resolveEntitiesIfNeeded( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nullable PolarisEntityCore resolvedEntity, @Nullable List otherTopLevelEntities) { @@ -282,13 +282,13 @@ private boolean resolveEntitiesIfNeeded( .collect(Collectors.toList()); // now lookup all these entities by name - Iterator activeRecordIt = + Iterator activeRecordIt = ms.lookupEntityActiveBatch(callCtx, entityActiveKeys).iterator(); // now validate if there was a change and if yes, re-resolve again for (PolarisEntityCore resolveEntity : toResolve) { // get associate active record - PolarisEntityActiveRecord activeEntityRecord = activeRecordIt.next(); + EntityNameLookupRecord activeEntityRecord = activeRecordIt.next(); // if this entity has been dropped (null) or replaced (<> ids), then fail validation if (activeEntityRecord == null || activeEntityRecord.getId() != resolveEntity.getId()) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisMetaStoreManagerImpl.java similarity index 87% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisMetaStoreManagerImpl.java index 98fd7789ca..da2ff2e924 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisMetaStoreManagerImpl.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -37,11 +37,10 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; @@ -51,6 +50,7 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.persistence.*; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -63,68 +63,12 @@ * and retrieve all Polaris metadata */ @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") -public class PolarisMetaStoreManagerImpl implements PolarisMetaStoreManager { +public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisMetaStoreManagerImpl.class); /** mapper, allows to serialize/deserialize properties to/from JSON */ private static final ObjectMapper MAPPER = new ObjectMapper(); - /** use synchronous drop for entities */ - private static final boolean USE_SYNCHRONOUS_DROP = true; - - /** - * Lookup an entity by its name - * - * @param callCtx call context - * @param ms meta store - * @param entityActiveKey lookup key - * @return the entity if it exists, null otherwise - */ - private @Nullable PolarisBaseEntity lookupEntityByName( - @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, - @Nonnull PolarisEntitiesActiveKey entityActiveKey) { - // ensure that the entity exists - PolarisEntityActiveRecord entityActiveRecord = ms.lookupEntityActive(callCtx, entityActiveKey); - - // if not found, return null - if (entityActiveRecord == null) { - return null; - } - - // lookup the entity, should be there - PolarisBaseEntity entity = - ms.lookupEntity(callCtx, entityActiveRecord.getCatalogId(), entityActiveRecord.getId()); - callCtx - .getDiagServices() - .checkNotNull( - entity, "unexpected_not_found_entity", "entityActiveRecord={}", entityActiveRecord); - - // return it now - return entity; - } - - /** - * Write this entity to the meta store. - * - * @param callCtx call context - * @param ms meta store in read/write mode - * @param entity entity to persist - * @param writeToActive if true, write it to active - */ - private void writeEntity( - @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, - @Nonnull PolarisBaseEntity entity, - boolean writeToActive) { - ms.writeToEntities(callCtx, entity); - ms.writeToEntitiesChangeTracking(callCtx, entity); - - if (writeToActive) { - ms.writeToEntitiesActive(callCtx, entity); - } - } - /** * Persist the specified new entity. Persist will write this entity in the ENTITIES, in the * ENTITIES_ACTIVE and finally in the ENTITIES_CHANGE_TRACKING tables @@ -135,7 +79,7 @@ private void writeEntity( */ private void persistNewEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity entity) { // validate the entity type and subtype @@ -186,7 +130,7 @@ private void persistNewEntity( entity.setToPurgeTimestamp(0); // write it - this.writeEntity(callCtx, ms, entity, true); + ms.writeEntity(callCtx, entity, true, null); } /** @@ -197,12 +141,16 @@ private void persistNewEntity( * @param callCtx call context * @param ms meta store * @param entity the entity which has been changed + * @param nameOrParentChanged indicates if parent or name changed + * @param originalEntity the original state of the entity before changes * @return the entity with its version and lastUpdateTimestamp updated */ private @Nonnull PolarisBaseEntity persistEntityAfterChange( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, - @Nonnull PolarisBaseEntity entity) { + @Nonnull TransactionalPersistence ms, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nonnull PolarisBaseEntity originalEntity) { // validate the entity type and subtype callCtx.getDiagServices().checkNotNull(entity, "unexpected_null_entity"); @@ -245,7 +193,7 @@ private void persistNewEntity( entity.setEntityVersion(entity.getEntityVersion() + 1); // persist it to the various slices - this.writeEntity(callCtx, ms, entity, false); + ms.writeEntity(callCtx, entity, nameOrParentChanged, originalEntity); // return it return entity; @@ -267,7 +215,7 @@ private void persistNewEntity( */ private void dropEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity entity) { // validate the entity type and subtype @@ -277,77 +225,52 @@ private void dropEntity( // creation timestamp must be filled callCtx.getDiagServices().check(entity.getDropTimestamp() == 0, "already_dropped"); - // delete it from active slice - ms.deleteFromEntitiesActive(callCtx, entity); - - // for now drop all entities synchronously - if (USE_SYNCHRONOUS_DROP) { - // use synchronous drop - - // delete ALL grant records to (if the entity is a grantee) and from that entity - final List grantsOnGrantee = - (entity.getType().isGrantee()) - ? ms.loadAllGrantRecordsOnGrantee(callCtx, entity.getCatalogId(), entity.getId()) - : List.of(); - final List grantsOnSecurable = - ms.loadAllGrantRecordsOnSecurable(callCtx, entity.getCatalogId(), entity.getId()); - ms.deleteAllEntityGrantRecords(callCtx, entity, grantsOnGrantee, grantsOnSecurable); - - // Now determine the set of entities on the other side of the grants we just removed. Grants - // from/to these entities has been removed, hence we need to update the grant version of - // each entity. Collect the id of each. - Set entityIdsGrantChanged = new HashSet<>(); - grantsOnGrantee.forEach( - gr -> - entityIdsGrantChanged.add( - new PolarisEntityId(gr.getSecurableCatalogId(), gr.getSecurableId()))); - grantsOnSecurable.forEach( - gr -> - entityIdsGrantChanged.add( - new PolarisEntityId(gr.getGranteeCatalogId(), gr.getGranteeId()))); - - // Bump up the grant version of these entities - List entities = - ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); - for (PolarisBaseEntity entityGrantChanged : entities) { - entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); - ms.writeToEntities(callCtx, entityGrantChanged); - ms.writeToEntitiesChangeTracking(callCtx, entityGrantChanged); - } - - // remove the entity being dropped now - ms.deleteFromEntities(callCtx, entity); - ms.deleteFromEntitiesChangeTracking(callCtx, entity); - - // if it is a principal, we also need to drop the secrets - if (entity.getType() == PolarisEntityType.PRINCIPAL) { - // get internal properties - Map properties = - this.deserializeProperties(callCtx, entity.getInternalProperties()); - - // get client_id - String clientId = properties.get(PolarisEntityConstants.getClientIdPropertyName()); - - // delete it from the secret slice - ms.deletePrincipalSecrets(callCtx, clientId, entity.getId()); - } - } else { + // for now drop all associated grants, etc. synchronously + // delete ALL grant records to (if the entity is a grantee) and from that entity + final List grantsOnGrantee = + (entity.getType().isGrantee()) + ? ms.loadAllGrantRecordsOnGrantee(callCtx, entity.getCatalogId(), entity.getId()) + : List.of(); + final List grantsOnSecurable = + ms.loadAllGrantRecordsOnSecurable(callCtx, entity.getCatalogId(), entity.getId()); + ms.deleteAllEntityGrantRecords(callCtx, entity, grantsOnGrantee, grantsOnSecurable); + + // Now determine the set of entities on the other side of the grants we just removed. Grants + // from/to these entities has been removed, hence we need to update the grant version of + // each entity. Collect the id of each. + Set entityIdsGrantChanged = new HashSet<>(); + grantsOnGrantee.forEach( + gr -> + entityIdsGrantChanged.add( + new PolarisEntityId(gr.getSecurableCatalogId(), gr.getSecurableId()))); + grantsOnSecurable.forEach( + gr -> + entityIdsGrantChanged.add( + new PolarisEntityId(gr.getGranteeCatalogId(), gr.getGranteeId()))); + + // Bump up the grant version of these entities + List entities = + ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); + for (PolarisBaseEntity entityGrantChanged : entities) { + PolarisBaseEntity originalEntity = new PolarisBaseEntity(entityGrantChanged); + entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); + ms.writeEntity(callCtx, entityGrantChanged, false, originalEntity); + } - // update the entity to indicate it has been dropped - final long now = System.currentTimeMillis(); - entity.setDropTimestamp(now); - entity.setLastUpdateTimestamp(now); + // remove the entity being dropped now + ms.deleteEntity(callCtx, entity); - // schedule purge - entity.setToPurgeTimestamp(now + PolarisEntityConstants.getRetentionTimeInMs()); + // if it is a principal, we also need to drop the secrets + if (entity.getType() == PolarisEntityType.PRINCIPAL) { + // get internal properties + Map properties = + this.deserializeProperties(callCtx, entity.getInternalProperties()); - // increment version - entity.setEntityVersion(entity.getEntityVersion() + 1); + // get client_id + String clientId = properties.get(PolarisEntityConstants.getClientIdPropertyName()); - // write to the dropped slice and to purge slice - ms.writeToEntities(callCtx, entity); - ms.writeToEntitiesDropped(callCtx, entity); - ms.writeToEntitiesChangeTracking(callCtx, entity); + // delete it from the secret slice + ms.deletePrincipalSecrets(callCtx, clientId, entity.getId()); } } @@ -364,7 +287,7 @@ private void dropEntity( */ private @Nonnull PolarisGrantRecord persistNewGrantRecord( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore securable, @Nonnull PolarisEntityCore grantee, @Nonnull PolarisPrivilege priv) { @@ -398,10 +321,11 @@ private void dropEntity( callCtx .getDiagServices() .checkNotNull(granteeEntity, "grantee_not_found", "grantee={}", grantee); + PolarisBaseEntity originalGranteeEntity = new PolarisBaseEntity(granteeEntity); // grants have changed, we need to bump-up the grants version granteeEntity.setGrantRecordsVersion(granteeEntity.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, granteeEntity, false); + ms.writeEntity(callCtx, granteeEntity, false, originalGranteeEntity); // we also need to invalidate the grants on that securable so that we can reload them. // load the securable and increment its grants version @@ -410,10 +334,11 @@ private void dropEntity( callCtx .getDiagServices() .checkNotNull(securableEntity, "securable_not_found", "securable={}", securable); + PolarisBaseEntity originalSecurableEntity = new PolarisBaseEntity(securableEntity); // grants have changed, we need to bump-up the grants version securableEntity.setGrantRecordsVersion(securableEntity.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, securableEntity, false); + ms.writeEntity(callCtx, securableEntity, false, originalSecurableEntity); // done, return the new grant record return grantRecord; @@ -431,7 +356,7 @@ private void dropEntity( */ private void revokeGrantRecord( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore securable, @Nonnull PolarisEntityCore grantee, @Nonnull PolarisGrantRecord grantRecord) { @@ -473,10 +398,11 @@ private void revokeGrantRecord( .getDiagServices() .checkNotNull( refreshGrantee, "missing_grantee", "grantRecord={} grantee={}", grantRecord, grantee); + PolarisBaseEntity originalRefreshGrantee = new PolarisBaseEntity(refreshGrantee); // grants have changed, we need to bump-up the grants version refreshGrantee.setGrantRecordsVersion(refreshGrantee.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, refreshGrantee, false); + ms.writeEntity(callCtx, refreshGrantee, false, originalRefreshGrantee); // we also need to invalidate the grants on that securable so that we can reload them. // load the securable and increment its grants version @@ -490,10 +416,11 @@ private void revokeGrantRecord( "grantRecord={} securable={}", grantRecord, securable); + PolarisBaseEntity originalRefreshSecurable = new PolarisBaseEntity(refreshSecurable); // grants have changed, we need to bump-up the grants version refreshSecurable.setGrantRecordsVersion(refreshSecurable.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, refreshSecurable, false); + ms.writeEntity(callCtx, refreshSecurable, false, originalRefreshSecurable); } /** @@ -513,7 +440,7 @@ private void revokeGrantRecord( */ private @Nonnull CreateCatalogResult createCatalog( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity catalog, @Nullable PolarisStorageIntegration integration, @Nonnull List principalRoles) { @@ -536,13 +463,13 @@ private void revokeGrantRecord( catalog); // lookup catalog admin role, should exist - PolarisEntitiesActiveKey adminRoleKey = - new PolarisEntitiesActiveKey( + PolarisBaseEntity catalogAdminRole = + ms.lookupEntityByName( + callCtx, refreshCatalog.getId(), refreshCatalog.getId(), PolarisEntityType.CATALOG_ROLE.getCode(), PolarisEntityConstants.getNameOfCatalogAdminRole()); - PolarisBaseEntity catalogAdminRole = this.lookupEntityByName(callCtx, ms, adminRoleKey); // if found, ensure not null callCtx @@ -555,16 +482,14 @@ private void revokeGrantRecord( } // check that a catalog with the same name does not exist already - PolarisEntitiesActiveKey catalogNameKey = - new PolarisEntitiesActiveKey( + // if it exists, this is an error, the client should retry + if (ms.lookupEntityIdAndSubTypeByName( + callCtx, PolarisEntityConstants.getNullId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.CATALOG.getCode(), - catalog.getName()); - PolarisEntityActiveRecord otherCatalogRecord = ms.lookupEntityActive(callCtx, catalogNameKey); - - // if it exists, this is an error, the client should retry - if (otherCatalogRecord != null) { + catalog.getName()) + != null) { return new CreateCatalogResult(BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, null); } @@ -597,14 +522,13 @@ private void revokeGrantRecord( // immediately assign its catalog_admin role if (principalRoles.isEmpty()) { // lookup service admin role, should exist - PolarisEntitiesActiveKey serviceAdminRoleKey = - new PolarisEntitiesActiveKey( + PolarisBaseEntity serviceAdminRole = + ms.lookupEntityByName( + callCtx, PolarisEntityConstants.getNullId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.PRINCIPAL_ROLE.getCode(), PolarisEntityConstants.getNameOfPrincipalServiceAdminRole()); - PolarisBaseEntity serviceAdminRole = - this.lookupEntityByName(callCtx, ms, serviceAdminRoleKey); callCtx.getDiagServices().checkNotNull(serviceAdminRole, "missing_service_admin_role"); this.persistNewGrantRecord( callCtx, ms, adminRole, serviceAdminRole, PolarisPrivilege.CATALOG_ROLE_USAGE); @@ -638,7 +562,7 @@ private void revokeGrantRecord( * @param ms meta store in read/write mode */ private void bootstrapPolarisService( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisMetaStoreSession ms) { + @Nonnull PolarisCallContext callCtx, @Nonnull TransactionalPersistence ms) { // Create a root container entity that can represent the securable for any top-level grants. PolarisBaseEntity rootContainer = @@ -700,7 +624,7 @@ private void bootstrapPolarisService( @Override public @Nonnull BaseResult bootstrapPolarisService(@Nonnull PolarisCallContext callCtx) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read/write transaction ms.runActionInTransaction(callCtx, () -> this.bootstrapPolarisService(callCtx, ms)); @@ -712,7 +636,7 @@ private void bootstrapPolarisService( @Override public @Nonnull BaseResult purge(@Nonnull PolarisCallContext callCtx) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read/write transaction LOGGER.warn("Deleting all metadata in the metastore..."); @@ -727,9 +651,9 @@ private void bootstrapPolarisService( * See {@link #readEntityByName(PolarisCallContext, List, PolarisEntityType, PolarisEntitySubType, * String)} */ - private @Nonnull PolarisMetaStoreManager.EntityResult readEntityByName( + private @Nonnull EntityResult readEntityByName( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityType entityType, @Nonnull PolarisEntitySubType entitySubType, @@ -743,10 +667,13 @@ private void bootstrapPolarisService( } // now looking the entity by name - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey( - resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType.getCode(), name); - PolarisBaseEntity entity = this.lookupEntityByName(callCtx, ms, entityActiveKey); + PolarisBaseEntity entity = + ms.lookupEntityByName( + callCtx, + resolver.getCatalogIdOrNull(), + resolver.getParentId(), + entityType.getCode(), + name); // if found, check if subType really matches if (entity != null @@ -763,14 +690,14 @@ private void bootstrapPolarisService( /** {@inheritDoc} */ @Override - public @Nonnull PolarisMetaStoreManager.EntityResult readEntityByName( + public @Nonnull EntityResult readEntityByName( @Nonnull PolarisCallContext callCtx, @Nullable List catalogPath, @Nonnull PolarisEntityType entityType, @Nonnull PolarisEntitySubType entitySubType, @Nonnull String name) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read/write transaction return ms.runInReadTransaction( @@ -782,7 +709,7 @@ private void bootstrapPolarisService( */ private @Nonnull ListEntitiesResult listEntities( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityType entityType, @Nonnull PolarisEntitySubType entitySubType) { @@ -795,9 +722,8 @@ private void bootstrapPolarisService( } // return list of active entities - List toreturnList = - ms.listActiveEntities( - callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType); + List toreturnList = + ms.listEntities(callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType); // prune the returned list with only entities matching the entity subtype if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { @@ -819,7 +745,7 @@ private void bootstrapPolarisService( @Nonnull PolarisEntityType entityType, @Nonnull PolarisEntitySubType entitySubType) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read transaction return ms.runInReadTransaction( @@ -830,7 +756,7 @@ private void bootstrapPolarisService( @Override public @Nonnull GenerateEntityIdResult generateNewEntityId(@Nonnull PolarisCallContext callCtx) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); return new GenerateEntityIdResult(ms.generateNewId(callCtx)); } @@ -878,7 +804,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** {@link #createPrincipal(PolarisCallContext, PolarisBaseEntity)} */ private @Nonnull CreatePrincipalResult createPrincipal( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity principal) { // validate input callCtx.getDiagServices().checkNotNull(principal, "unexpected_null_principal"); @@ -940,17 +866,14 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // check that a principal with the same name does not exist already - PolarisEntitiesActiveKey principalNameKey = - new PolarisEntitiesActiveKey( + // if it exists, this is an error, the client should retry + if (ms.lookupEntityIdAndSubTypeByName( + callCtx, PolarisEntityConstants.getNullId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.PRINCIPAL.getCode(), - principal.getName()); - PolarisEntityActiveRecord otherPrincipalRecord = - ms.lookupEntityActive(callCtx, principalNameKey); - - // if it exists, this is an error, the client should retry - if (otherPrincipalRecord != null) { + principal.getName()) + != null) { return new CreatePrincipalResult(BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, null); } @@ -978,7 +901,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str public @Nonnull CreatePrincipalResult createPrincipal( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity principal) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction(callCtx, () -> this.createPrincipal(callCtx, ms, principal)); @@ -986,7 +909,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** See {@link #loadPrincipalSecrets(PolarisCallContext, String)} */ private @Nullable PolarisPrincipalSecrets loadPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, PolarisMetaStoreSession ms, @Nonnull String clientId) { + @Nonnull PolarisCallContext callCtx, TransactionalPersistence ms, @Nonnull String clientId) { return ms.loadPrincipalSecrets(callCtx, clientId); } @@ -995,7 +918,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str public @Nonnull PrincipalSecretsResult loadPrincipalSecrets( @Nonnull PolarisCallContext callCtx, @Nonnull String clientId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction PolarisPrincipalSecrets secrets = @@ -1009,7 +932,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** See {@link #} */ private @Nullable PolarisPrincipalSecrets rotatePrincipalSecrets( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull String clientId, long principalId, boolean reset, @@ -1022,6 +945,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } PolarisBaseEntity principal = loadEntityResult.getEntity(); + PolarisBaseEntity originalPrincipal = new PolarisBaseEntity(principal); Map internalProps = PolarisObjectMapperUtil.deserializeProperties( callCtx, @@ -1043,14 +967,14 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str principal.setInternalProperties( PolarisObjectMapperUtil.serializeProperties(callCtx, internalProps)); principal.setEntityVersion(principal.getEntityVersion() + 1); - writeEntity(callCtx, ms, principal, true); + ms.writeEntity(callCtx, principal, true, originalPrincipal); } else if (internalProps.containsKey( PolarisEntityConstants.PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE)) { internalProps.remove(PolarisEntityConstants.PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE); principal.setInternalProperties( PolarisObjectMapperUtil.serializeProperties(callCtx, internalProps)); principal.setEntityVersion(principal.getEntityVersion() + 1); - writeEntity(callCtx, ms, principal, true); + ms.writeEntity(callCtx, principal, true, originalPrincipal); } return secrets; } @@ -1064,7 +988,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str boolean reset, @Nonnull String oldSecretHash) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction PolarisPrincipalSecrets secrets = @@ -1086,7 +1010,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nonnull PolarisBaseEntity catalog, @Nonnull List principalRoles) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); Map internalProp = getInternalPropertyMap(callCtx, catalog); String integrationIdentifierOrId = @@ -1115,7 +1039,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** {@link #createEntityIfNotExists(PolarisCallContext, List, PolarisBaseEntity)} */ private @Nonnull EntityResult createEntityIfNotExists( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { @@ -1141,13 +1065,13 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // check if an entity does not already exist with the same name. If true, this is an error - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey( + EntityNameLookupRecord entityActiveRecord = + ms.lookupEntityIdAndSubTypeByName( + callCtx, entity.getCatalogId(), entity.getParentId(), entity.getType().getCode(), entity.getName()); - PolarisEntityActiveRecord entityActiveRecord = ms.lookupEntityActive(callCtx, entityActiveKey); if (entityActiveRecord != null) { return new EntityResult( BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, entityActiveRecord.getSubTypeCode()); @@ -1167,7 +1091,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1180,7 +1104,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List catalogPath, @Nonnull List entities) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1207,7 +1131,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull EntityResult updateEntityPropertiesIfNotChanged( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { // entity cannot be null @@ -1233,13 +1157,16 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str return new EntityResult(BaseResult.ReturnStatus.TARGET_ENTITY_CONCURRENTLY_MODIFIED, null); } + PolarisBaseEntity originalEntity = new PolarisBaseEntity(entityRefreshed); + // update the two properties entityRefreshed.setInternalProperties(entity.getInternalProperties()); entityRefreshed.setProperties(entity.getProperties()); // persist this entity after changing it. This will update the version and update the last // updated time. Because the entity version is changed, we will update the change tracking table - PolarisBaseEntity persistedEntity = this.persistEntityAfterChange(callCtx, ms, entityRefreshed); + PolarisBaseEntity persistedEntity = + this.persistEntityAfterChange(callCtx, ms, entityRefreshed, false, originalEntity); return new EntityResult(persistedEntity); } @@ -1250,7 +1177,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1260,7 +1187,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** See {@link #updateEntitiesPropertiesIfNotChanged(PolarisCallContext, List)} */ private @Nonnull EntitiesResult updateEntitiesPropertiesIfNotChanged( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull List entities) { // ensure that the entities list is not null callCtx.getDiagServices().checkNotNull(entities, "unexpected_null_entities"); @@ -1295,7 +1222,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str public @Nonnull EntitiesResult updateEntitiesPropertiesIfNotChanged( @Nonnull PolarisCallContext callCtx, @Nonnull List entities) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1308,7 +1235,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull EntityResult renameEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityCore entityToRename, @Nullable List newCatalogPath, @@ -1371,21 +1298,22 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // ensure that nothing exists where we create that entity - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey( + // if this entity already exists, this is an error + EntityNameLookupRecord entityActiveRecord = + ms.lookupEntityIdAndSubTypeByName( + callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), refreshEntityToRename.getTypeCode(), renamedEntity.getName()); - // if this entity already exists, this is an error - PolarisEntityActiveRecord entityActiveRecord = ms.lookupEntityActive(callCtx, entityActiveKey); if (entityActiveRecord != null) { return new EntityResult( BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, entityActiveRecord.getSubTypeCode()); } - // all good, delete the existing entity from the active slice - ms.deleteFromEntitiesActive(callCtx, refreshEntityToRename); + // Create a copy of the original before we change its fields so that we can pass in the + // old version for the persistence layer to work out whether to unlink previous name-lookups + PolarisBaseEntity originalEntity = new PolarisBaseEntity(refreshEntityToRename); // change its name now refreshEntityToRename.setName(renamedEntity.getName()); @@ -1397,13 +1325,11 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str refreshEntityToRename.setParentId(resolver.getParentId()); } - // persist back to the active slice with its new name and parent - ms.writeToEntitiesActive(callCtx, refreshEntityToRename); - - // persist the entity after change. This wil update the lastUpdateTimestamp and bump up the - // version + // persist the entity after change. This will update the lastUpdateTimestamp and bump up the + // version. Indicate that the nameOrParent changed, so so that we also update any by-name + // lookups if applicable PolarisBaseEntity renamedEntityToReturn = - this.persistEntityAfterChange(callCtx, ms, refreshEntityToRename); + this.persistEntityAfterChange(callCtx, ms, refreshEntityToRename, true, originalEntity); return new EntityResult(renamedEntityToReturn); } @@ -1416,7 +1342,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List newCatalogPath, @Nonnull PolarisEntity renamedEntity) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1433,7 +1359,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull DropEntityResult dropEntityIfExists( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityCore entityToDrop, @Nullable Map cleanupProperties, @@ -1477,7 +1403,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str // get the list of catalog roles, at most 2 List catalogRoles = - ms.listActiveEntities( + ms.listEntities( callCtx, catalogId, catalogId, @@ -1548,7 +1474,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable Map cleanupProperties, boolean cleanup) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1572,7 +1498,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull PolarisEntityResolver resolveRoleToGranteeUsageGrant( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable PolarisEntityCore catalog, @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { @@ -1622,7 +1548,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore grantee, @Nullable List catalogPath, @Nonnull PolarisEntityCore securable) { @@ -1649,7 +1575,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult grantUsageOnRoleToGrantee( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable PolarisEntityCore catalog, @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { @@ -1686,7 +1612,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1699,7 +1625,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult revokeUsageOnRoleFromGrantee( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable PolarisEntityCore catalog, @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { @@ -1748,7 +1674,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1761,7 +1687,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult grantPrivilegeOnSecurableToRole( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore grantee, @Nullable List catalogPath, @Nonnull PolarisEntityCore securable, @@ -1791,7 +1717,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore securable, @Nonnull PolarisPrivilege privilege) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1807,7 +1733,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult revokePrivilegeOnSecurableFromRole( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore grantee, @Nullable List catalogPath, @Nonnull PolarisEntityCore securable, @@ -1853,7 +1779,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore securable, @Nonnull PolarisPrivilege privilege) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1866,7 +1792,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** {@link #loadGrantsOnSecurable(PolarisCallContext, long, long)} */ private @Nonnull LoadGrantsResult loadGrantsOnSecurable( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long securableCatalogId, long securableId) { @@ -1906,7 +1832,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull LoadGrantsResult loadGrantsOnSecurable( @Nonnull PolarisCallContext callCtx, long securableCatalogId, long securableId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -1916,7 +1842,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** {@link #loadGrantsToGrantee(PolarisCallContext, long, long)} */ public @Nonnull LoadGrantsResult loadGrantsToGrantee( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long granteeCatalogId, long granteeId) { @@ -1955,7 +1881,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull LoadGrantsResult loadGrantsToGrantee( @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -1965,7 +1891,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** {@link PolarisMetaStoreManager#loadEntitiesChangeTracking(PolarisCallContext, List)} */ private @Nonnull ChangeTrackingResult loadEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull List entityIds) { List changeTracking = ms.lookupEntityVersions(callCtx, entityIds); @@ -1977,7 +1903,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull ChangeTrackingResult loadEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull List entityIds) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -1987,7 +1913,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** Refer to {@link #loadEntity(PolarisCallContext, long, long)} */ private @Nonnull EntityResult loadEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long entityCatalogId, long entityId) { // this is an easy one @@ -2002,7 +1928,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull EntityResult loadEntity( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -2012,13 +1938,13 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** Refer to {@link #loadTasks(PolarisCallContext, String, int)} */ private @Nonnull EntitiesResult loadTasks( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, String executorId, int limit) { // find all available tasks List availableTasks = - ms.listActiveEntities( + ms.listEntities( callCtx, PolarisEntityConstants.getRootEntityId(), PolarisEntityConstants.getRootEntityId(), @@ -2042,6 +1968,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( availableTasks.forEach( task -> { + PolarisBaseEntity originalTask = new PolarisBaseEntity(task); Map properties = PolarisObjectMapperUtil.deserializeProperties(callCtx, task.getProperties()); properties.put(PolarisTaskConstants.LAST_ATTEMPT_EXECUTOR_ID, executorId); @@ -2055,7 +1982,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( + 1)); task.setEntityVersion(task.getEntityVersion() + 1); task.setProperties(PolarisObjectMapperUtil.serializeProperties(callCtx, properties)); - writeEntity(callCtx, ms, task, false); + ms.writeEntity(callCtx, task, false, originalTask); }); return new EntitiesResult(availableTasks); } @@ -2063,7 +1990,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Override public @Nonnull EntitiesResult loadTasks( @Nonnull PolarisCallContext callCtx, String executorId, int limit) { - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); return ms.runInTransaction(callCtx, () -> this.loadTasks(callCtx, ms, executorId, limit)); } @@ -2078,7 +2005,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull Set allowedWriteLocations) { // get meta store session we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); callCtx .getDiagServices() .check( @@ -2107,7 +2034,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( entityId); PolarisStorageConfigurationInfo storageConfigurationInfo = - readStorageConfiguration(callCtx, reloadedEntity.getEntity()); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity()); try { EnumMap creds = storageIntegration.getSubscopedCreds( @@ -2132,7 +2059,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull Set actions, @Nonnull Set locations) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); callCtx .getDiagServices() .check( @@ -2159,7 +2086,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( // validate access PolarisStorageConfigurationInfo storageConfigurationInfo = - readStorageConfiguration(callCtx, reloadedEntity.getEntity()); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity()); Map validateLocationAccess = storageIntegration .validateAccessToLocations(storageConfigurationInfo, actions, locations) @@ -2174,26 +2101,6 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( return new ValidateAccessResult(validateLocationAccess); } - public static PolarisStorageConfigurationInfo readStorageConfiguration( - @Nonnull PolarisCallContext callCtx, PolarisBaseEntity reloadedEntity) { - Map propMap = - PolarisObjectMapperUtil.deserializeProperties( - callCtx, reloadedEntity.getInternalProperties()); - String storageConfigInfoStr = - propMap.get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); - - callCtx - .getDiagServices() - .check( - storageConfigInfoStr != null, - "missing_storage_configuration_info", - "catalogId={}, entityId={}", - reloadedEntity.getCatalogId(), - reloadedEntity.getId()); - return PolarisStorageConfigurationInfo.deserialize( - callCtx.getDiagServices(), storageConfigInfoStr); - } - /** * Get the internal property map for an entity * @@ -2214,7 +2121,7 @@ public Map getInternalPropertyMap( /** {@link #loadResolvedEntityById(PolarisCallContext, long, long)} */ private @Nonnull ResolvedEntityResult loadResolvedEntityById( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long entityCatalogId, long entityId) { @@ -2245,7 +2152,7 @@ public Map getInternalPropertyMap( public @Nonnull ResolvedEntityResult loadResolvedEntityById( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -2255,16 +2162,15 @@ public Map getInternalPropertyMap( /** {@link #loadResolvedEntityById(PolarisCallContext, long, long)} */ private @Nonnull ResolvedEntityResult loadResolvedEntityByName( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long entityCatalogId, long parentId, @Nonnull PolarisEntityType entityType, @Nonnull String entityName) { // load that entity - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey(entityCatalogId, parentId, entityType.getCode(), entityName); - PolarisBaseEntity entity = this.lookupEntityByName(callCtx, ms, entityActiveKey); + PolarisBaseEntity entity = + ms.lookupEntityByName(callCtx, entityCatalogId, parentId, entityType.getCode(), entityName); // null if entity not found if (entity == null) { @@ -2296,7 +2202,7 @@ public Map getInternalPropertyMap( @Nonnull PolarisEntityType entityType, @Nonnull String entityName) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction ResolvedEntityResult result = @@ -2323,14 +2229,13 @@ public Map getInternalPropertyMap( EntityResult backfillResult = this.createEntityIfNotExists(callCtx, ms, null, rootContainer); if (backfillResult.isSuccess()) { - PolarisEntitiesActiveKey serviceAdminRoleKey = - new PolarisEntitiesActiveKey( + PolarisBaseEntity serviceAdminRole = + ms.lookupEntityByName( + callCtx, 0L, 0L, PolarisEntityType.PRINCIPAL_ROLE.getCode(), PolarisEntityConstants.getNameOfPrincipalServiceAdminRole()); - PolarisBaseEntity serviceAdminRole = - this.lookupEntityByName(callCtx, ms, serviceAdminRoleKey); if (serviceAdminRole != null) { this.persistNewGrantRecord( callCtx, @@ -2356,7 +2261,7 @@ public Map getInternalPropertyMap( /** {@inheritDoc} */ private @Nonnull ResolvedEntityResult refreshResolvedEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, int entityVersion, int entityGrantRecordsVersion, @Nonnull PolarisEntityType entityType, @@ -2415,7 +2320,7 @@ public Map getInternalPropertyMap( long entityCatalogId, long entityId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapMetaStoreSessionImpl.java similarity index 90% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapMetaStoreSessionImpl.java index bea2054156..312f3576b9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapMetaStoreSessionImpl.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import com.google.common.base.Predicates; import jakarta.annotation.Nonnull; @@ -27,20 +27,22 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.BaseMetaStoreManager; +import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -public class PolarisTreeMapMetaStoreSessionImpl implements PolarisMetaStoreSession { +public class PolarisTreeMapMetaStoreSessionImpl extends TransactionalPersistence { // the TreeMap store to use private final PolarisTreeMapStore store; @@ -126,15 +128,6 @@ public void writeToEntitiesActive( this.store.getSliceEntitiesActive().write(entity); } - /** {@inheritDoc} */ - @Override - public void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // write it - this.store.getSliceEntitiesDropped().write(entity); - this.store.getSliceEntitiesDroppedToPurge().write(entity); - } - /** {@inheritDoc} */ @Override public void writeToEntitiesChangeTracking( @@ -169,15 +162,6 @@ public void deleteFromEntitiesActive( this.store.getSliceEntitiesActive().delete(this.store.buildEntitiesActiveKey(entity)); } - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // delete it - this.store.getSliceEntitiesDropped().delete(entity); - this.store.getSliceEntitiesDroppedToPurge().delete(entity); - } - /** * {@inheritDoc} * @@ -248,18 +232,6 @@ public void deleteAll(@Nonnull PolarisCallContext callCtx) { .collect(Collectors.toList()); } - /** {@inheritDoc} */ - @Override - public int lookupEntityVersion( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId) { - PolarisBaseEntity baseEntity = - this.store - .getSliceEntitiesChangeTracking() - .read(this.store.buildKeyComposite(catalogId, entityId)); - - return baseEntity == null ? 0 : baseEntity.getEntityVersion(); - } - /** {@inheritDoc} */ @Override public @Nonnull List lookupEntityVersions( @@ -283,7 +255,7 @@ public int lookupEntityVersion( /** {@inheritDoc} */ @Override @Nullable - public PolarisEntityActiveRecord lookupEntityActive( + public EntityNameLookupRecord lookupEntityActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey) { // lookup the active entity slice PolarisBaseEntity entity = @@ -299,7 +271,7 @@ public PolarisEntityActiveRecord lookupEntityActive( // return record return (entity == null) ? null - : new PolarisEntityActiveRecord( + : new EntityNameLookupRecord( entity.getCatalogId(), entity.getId(), entity.getParentId(), @@ -311,7 +283,7 @@ public PolarisEntityActiveRecord lookupEntityActive( /** {@inheritDoc} */ @Override @Nonnull - public List lookupEntityActiveBatch( + public List lookupEntityActiveBatch( @Nonnull PolarisCallContext callCtx, @Nonnull List entityActiveKeys) { // now build a list to quickly verify that nothing has changed @@ -322,23 +294,23 @@ public List lookupEntityActiveBatch( /** {@inheritDoc} */ @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); + return listEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType, @Nonnull Predicate entityFilter) { // full range scan under the parent for that type - return listActiveEntities( + return listEntities( callCtx, catalogId, parentId, @@ -346,7 +318,7 @@ public List lookupEntityActiveBatch( Integer.MAX_VALUE, entityFilter, entity -> - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( entity.getCatalogId(), entity.getId(), entity.getParentId(), @@ -356,7 +328,7 @@ public List lookupEntityActiveBatch( } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @@ -565,7 +537,7 @@ PolarisStorageIntegration createStorageIntegration( PolarisStorageIntegration loadPolarisStorageIntegration( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { PolarisStorageConfigurationInfo storageConfig = - PolarisMetaStoreManagerImpl.readStorageConfiguration(callCtx, entity); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, entity); return storageIntegrationProvider.getStorageIntegrationForConfig(storageConfig); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapStore.java similarity index 90% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapStore.java index 544bcf0ffe..fe80bdf23a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapStore.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import jakarta.annotation.Nonnull; import java.util.ArrayList; @@ -194,16 +194,10 @@ public boolean isWrite() { // all entities private final Slice sliceEntities; - // all entities + // all entities by-name private final Slice sliceEntitiesActive; - // all entities dropped - private final Slice sliceEntitiesDropped; - - // all entities dropped - private final Slice sliceEntitiesDroppedToPurge; - - // all entities dropped + // all entities just holding their entityVersions and grantVersions private final Slice sliceEntitiesChangeTracking; // all grant records indexed by securable @@ -231,32 +225,9 @@ public PolarisTreeMapStore(@Nonnull PolarisDiagnostics diagnostics) { entity -> String.format("%d::%d", entity.getCatalogId(), entity.getId()), PolarisBaseEntity::new); - // the entities active slice + // the entities active slice; simply acts as a name-based index into the entities slice this.sliceEntitiesActive = new Slice<>(this::buildEntitiesActiveKey, PolarisBaseEntity::new); - // the entities active slice - this.sliceEntitiesDropped = - new Slice<>( - entity -> - String.format( - "%d::%d::%s::%d::%d::%d", - entity.getCatalogId(), - entity.getParentId(), - entity.getName(), - entity.getTypeCode(), - entity.getSubTypeCode(), - entity.getDropTimestamp()), - PolarisBaseEntity::new); - - // the entities active slice - this.sliceEntitiesDroppedToPurge = - new Slice<>( - entity -> - String.format( - "%d::%d::%s", - entity.getToPurgeTimestamp(), entity.getCatalogId(), entity.getId()), - PolarisBaseEntity::new); - // change tracking this.sliceEntitiesChangeTracking = new Slice<>( @@ -370,8 +341,6 @@ private void startWriteTransaction() { this.tr = new Transaction(true); this.sliceEntities.startWriteTransaction(); this.sliceEntitiesActive.startWriteTransaction(); - this.sliceEntitiesDropped.startWriteTransaction(); - this.sliceEntitiesDroppedToPurge.startWriteTransaction(); this.sliceEntitiesChangeTracking.startWriteTransaction(); this.sliceGrantRecords.startWriteTransaction(); this.sliceGrantRecordsByGrantee.startWriteTransaction(); @@ -382,8 +351,6 @@ private void startWriteTransaction() { void rollback() { this.sliceEntities.rollback(); this.sliceEntitiesActive.rollback(); - this.sliceEntitiesDropped.rollback(); - this.sliceEntitiesDroppedToPurge.rollback(); this.sliceEntitiesChangeTracking.rollback(); this.sliceGrantRecords.rollback(); this.sliceGrantRecordsByGrantee.rollback(); @@ -510,14 +477,6 @@ public Slice getSliceEntitiesActive() { return sliceEntitiesActive; } - public Slice getSliceEntitiesDropped() { - return sliceEntitiesDropped; - } - - public Slice getSliceEntitiesDroppedToPurge() { - return sliceEntitiesDroppedToPurge; - } - public Slice getSliceEntitiesChangeTracking() { return sliceEntitiesChangeTracking; } @@ -548,8 +507,6 @@ void deleteAll() { this.ensureReadWriteTr(); this.sliceEntities.deleteAll(); this.sliceEntitiesActive.deleteAll(); - this.sliceEntitiesDropped.deleteAll(); - this.sliceEntitiesDroppedToPurge.deleteAll(); this.sliceEntitiesChangeTracking.deleteAll(); this.sliceGrantRecordsByGrantee.deleteAll(); this.sliceGrantRecords.deleteAll(); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java new file mode 100644 index 0000000000..856fc79210 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence.transactional; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Supplier; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.IntegrationPersistence; + +/** + * Extends BasePersistence to express a more "transaction-oriented" control flow for backing stores + * which can support a runInTransaction semantic, while providing default implementations of some of + * the BasePersistence methods in terms of lower-level methods that subclasses must implement. + */ +public abstract class TransactionalPersistence implements BasePersistence, IntegrationPersistence { + + /** + * Run the specified transaction code (a Supplier lambda type) in a database read/write + * transaction. If the code of the transaction does not throw any exception and returns normally, + * the transaction will be committed, else the transaction will be automatically rolled-back on + * error. The result of the supplier lambda is returned if success, else the error will be + * re-thrown. + * + * @param callCtx call context + * @param transactionCode code of the transaction being executed, a supplier lambda + */ + public abstract T runInTransaction( + @Nonnull PolarisCallContext callCtx, @Nonnull Supplier transactionCode); + + /** + * Run the specified transaction code (a runnable lambda type) in a database read/write + * transaction. If the code of the transaction does not throw any exception and returns normally, + * the transaction will be committed, else the transaction will be automatically rolled-back on + * error. + * + * @param callCtx call context + * @param transactionCode code of the transaction being executed, a runnable lambda + */ + public abstract void runActionInTransaction( + @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); + + /** + * Run the specified transaction code (a Supplier lambda type) in a database read transaction. If + * the code of the transaction does not throw any exception and returns normally, the transaction + * will be committed, else the transaction will be automatically rolled-back on error. The result + * of the supplier lambda is returned if success, else the error will be re-thrown. + * + * @param callCtx call context + * @param transactionCode code of the transaction being executed, a supplier lambda + */ + public abstract T runInReadTransaction( + @Nonnull PolarisCallContext callCtx, @Nonnull Supplier transactionCode); + + /** + * Run the specified transaction code (a runnable lambda type) in a database read transaction. If + * the code of the transaction does not throw any exception and returns normally, the transaction + * will be committed, else the transaction will be automatically rolled-back on error. + * + * @param callCtx call context + * @param transactionCode code of the transaction being executed, a runnable lambda + */ + public abstract void runActionInReadTransaction( + @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); + + /** {@inheritDoc} */ + @Override + public void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List entities, + @Nullable List originalEntities) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + /** {@inheritDoc} */ + @Override + public PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + // TODO: Consistently pull down the runInTransaction logic without running into conflicting + // nested transactions into here so that instead of having the caller be responsible for + // initiating the runInReadTransaction, we make this method call inherently safe to do + // the two-phase lookup. + PolarisEntitiesActiveKey entityActiveKey = + new PolarisEntitiesActiveKey(catalogId, parentId, typeCode, name); + + // ensure that the entity exists + EntityNameLookupRecord entityActiveRecord = lookupEntityActive(callCtx, entityActiveKey); + + // if not found, return null + if (entityActiveRecord == null) { + return null; + } + + // lookup the entity, should be there + PolarisBaseEntity entity = + lookupEntity(callCtx, entityActiveRecord.getCatalogId(), entityActiveRecord.getId()); + callCtx + .getDiagServices() + .checkNotNull( + entity, "unexpected_not_found_entity", "entityActiveRecord={}", entityActiveRecord); + + // return it now + return entity; + } + + /** {@inheritDoc} */ + @Override + public EntityNameLookupRecord lookupEntityIdAndSubTypeByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + PolarisEntitiesActiveKey entityActiveKey = + new PolarisEntitiesActiveKey(catalogId, parentId, typeCode, name); + return lookupEntityActive(callCtx, entityActiveKey); + } + + /** + * Lookup an entity by entityActiveKey + * + * @param callCtx call context + * @param entityActiveKey key by name + * @return null if the specified entity does not exist or has been dropped. + */ + @Nullable + protected abstract EntityNameLookupRecord lookupEntityActive( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); + + /** + * Lookup the specified set of entities by entityActiveKeys Return the result, a parallel list of + * active records. A record in that list will be null if its associated lookup failed + * + * @return the list of entityActiveKeys for the specified lookup operation + */ + @Nonnull + public abstract List lookupEntityActiveBatch( + @Nonnull PolarisCallContext callCtx, List entityActiveKeys); + + /** {@inheritDoc} */ + @Override + public void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity) { + // TODO: Pull down relevant compare-and-swap semantics from PolarisMetaStoreManagerImpl + // into this layer. + writeToEntities(callCtx, entity); + writeToEntitiesChangeTracking(callCtx, entity); + + if (nameOrParentChanged) { + if (originalEntity != null) { + // In our case, rename isn't automatically handled when the main "entities" slice + // is updated; instead we must explicitly remove from the old entitiesActive + // key as well. + deleteFromEntitiesActive(callCtx, originalEntity); + } + writeToEntitiesActive(callCtx, entity); + } + } + + /** + * Write the base entity to the entities table. If there is a conflict (existing record with the + * same id), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param entity entity record to write, potentially replacing an existing entity record with the + * same key + */ + protected abstract void writeToEntities( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Write the base entity to the entities_active table. If there is a conflict (existing record + * with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param entity entity record to write, potentially replacing an existing entity record with the + * same key + */ + protected abstract void writeToEntitiesActive( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Write the base entity to the entities change tracking table. If there is a conflict (existing + * record with the same id), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param entity entity record to write, potentially replacing an existing entity record with the + * same key + */ + protected abstract void writeToEntitiesChangeTracking( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** {@inheritDoc} */ + @Override + public void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { + deleteFromEntitiesActive(callCtx, entity); + deleteFromEntities(callCtx, entity); + deleteFromEntitiesChangeTracking(callCtx, entity); + } + + /** + * Delete the base entity from the entities table. + * + * @param callCtx call context + * @param entity entity record to delete + */ + protected abstract void deleteFromEntities( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); + + /** + * Delete the base entity from the entities_active table. + * + * @param callCtx call context + * @param entity entity record to delete + */ + protected abstract void deleteFromEntitiesActive( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); + + /** + * Delete the base entity from the entities change tracking table + * + * @param callCtx call context + * @param entity entity record to delete + */ + protected abstract void deleteFromEntitiesChangeTracking( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); + + /** Rollback the current transaction */ + public abstract void rollback(); +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java index 8dc3c84aba..d6015160ee 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java @@ -33,6 +33,10 @@ import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -47,7 +51,7 @@ public class EntityCacheTest { private final PolarisTreeMapStore store; // to interact with the metastore - private final PolarisMetaStoreSession metaStore; + private final TransactionalPersistence metaStore; // polaris call context private final PolarisCallContext callCtx; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java index e44b455777..65f8080d91 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java @@ -25,6 +25,9 @@ import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.mockito.Mockito; public class PolarisTreeMapMetaStoreManagerTest extends BasePolarisMetaStoreManagerTest { diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index 5270a90e22..c3e5f75327 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -49,6 +49,10 @@ import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -63,7 +67,7 @@ public class ResolverTest { private final PolarisTreeMapStore store; // to interact with the metastore - private final PolarisMetaStoreSession metaStore; + private final TransactionalPersistence metaStore; // polaris call context private final PolarisCallContext callCtx; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index d19bc41c8b..848367fe5c 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -39,11 +39,10 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.BaseResult; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; -import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; -import org.apache.polaris.core.persistence.PolarisTreeMapStore; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisCredentialVendor.ScopedCredentialsResult; import org.assertj.core.api.Assertions; @@ -67,10 +66,10 @@ public StorageCredentialCacheTest() { // the entity store, use treemap implementation PolarisTreeMapStore store = new PolarisTreeMapStore(diagServices); // to interact with the metastore - PolarisMetaStoreSession metaStore = + TransactionalPersistence metaStore = new PolarisTreeMapMetaStoreSessionImpl(store, Mockito.mock(), RANDOM_SECRETS); callCtx = new PolarisCallContext(metaStore, diagServices); - metaStoreManager = Mockito.mock(PolarisMetaStoreManagerImpl.class); + metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); storageCredentialCache = new StorageCredentialCache(); } diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 1b5483afe5..34b325160c 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -37,9 +37,9 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.TaskEntity; @@ -122,7 +122,7 @@ void testCreateEntities() { .extracting(PolarisEntity::toCore) .containsExactly(PolarisEntity.toCore(task1), PolarisEntity.toCore(task2)); - List listedEntities = + List listedEntities = metaStoreManager .listEntities( polarisTestMetaStoreManager.polarisCallContext, @@ -134,14 +134,14 @@ void testCreateEntities() { .isNotNull() .hasSize(2) .containsExactly( - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( task1.getCatalogId(), task1.getId(), task1.getParentId(), task1.getName(), task1.getTypeCode(), task1.getSubTypeCode()), - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( task2.getCatalogId(), task2.getId(), task2.getParentId(), diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 0c7c4131da..5270990a9a 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -30,10 +30,10 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisGrantManager.LoadGrantsResult; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; @@ -657,7 +657,7 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD path.add(entityToDrop); // get all children, cannot be null - List children = + List children = polarisMetaStoreManager .listEntities( this.polarisCallContext, @@ -1189,7 +1189,7 @@ private void validateListReturn( List> expectedResult) { // list the entities under the specified path - List result = + List result = polarisMetaStoreManager .listEntities(this.polarisCallContext, path, entityType, entitySubType) .getEntities(); @@ -1201,7 +1201,7 @@ private void validateListReturn( // ensure all elements are found for (Pair expected : expectedResult) { boolean found = false; - for (PolarisEntityActiveRecord res : result) { + for (EntityNameLookupRecord res : result) { if (res.getName().equals(expected.getLeft()) && expected.getRight().getCode() == res.getSubTypeCode()) { found = true; @@ -1498,7 +1498,7 @@ private void refreshCacheEntry( /** validate that the root catalog was properly constructed */ void validateBootstrap() { // load all principals - List principals = + List principals = polarisMetaStoreManager .listEntities( this.polarisCallContext, @@ -1511,7 +1511,7 @@ void validateBootstrap() { Assertions.assertThat(principals).isNotNull().hasSize(1); // get catalog list information - PolarisEntityActiveRecord principalListInfo = principals.get(0); + EntityNameLookupRecord principalListInfo = principals.get(0); // now make sure this principal was properly persisted PolarisBaseEntity principal = @@ -1524,7 +1524,7 @@ void validateBootstrap() { PolarisEntitySubType.NULL_SUBTYPE); // load all principal roles - List principalRoles = + List principalRoles = polarisMetaStoreManager .listEntities( this.polarisCallContext, @@ -1537,7 +1537,7 @@ void validateBootstrap() { Assertions.assertThat(principalRoles).isNotNull().hasSize(1); // get catalog list information - PolarisEntityActiveRecord roleListInfo = principalRoles.get(0); + EntityNameLookupRecord roleListInfo = principalRoles.get(0); // now make sure this principal role was properly persisted PolarisBaseEntity principalRole = diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index a3eb9b1929..cbe1cc2627 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -44,8 +44,8 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.auth.Authenticator; import org.apache.polaris.service.auth.TokenBrokerFactory; @@ -111,7 +111,7 @@ public PolarisCallContext polarisCallContext( PolarisConfigurationStore configurationStore, MetaStoreManagerFactory metaStoreManagerFactory, Clock clock) { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); return new PolarisCallContext(metaStoreSession, diagServices, configurationStore, clock); } @@ -225,7 +225,7 @@ public PolarisMetaStoreManager polarisMetaStoreManager( @Produces @RequestScoped - public PolarisMetaStoreSession polarisMetaStoreSession( + public TransactionalPersistence polarisMetaStoreSession( RealmContext realmContext, MetaStoreManagerFactory metaStoreManagerFactory) { return metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index e6670b4087..e1176ce515 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -71,8 +71,8 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; @@ -177,7 +177,7 @@ public Map getConfigOverrides() { protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; protected PolarisMetaStoreManager metaStoreManager; - protected PolarisMetaStoreSession metaStoreSession; + protected TransactionalPersistence metaStoreSession; protected PolarisBaseEntity catalogEntity; protected PrincipalEntity principalEntity; protected CallContext callContext; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 370ede6623..49d976fe66 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -86,10 +86,10 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -328,7 +328,7 @@ public PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmCon } @Override - public Supplier getOrCreateSessionSupplier( + public Supplier getOrCreateSessionSupplier( RealmContext realmContext) { return () -> polarisContext.getMetaStore(); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java index 06901b0f13..ad2d0f055c 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java @@ -49,8 +49,8 @@ import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; @@ -176,7 +176,7 @@ public void testTableCleanup() throws IOException { @Test public void testTableCleanupHandlesAlreadyDeletedMetadata() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO() { @@ -230,7 +230,7 @@ public void close() { @Test public void testTableCleanupDuplicatesTasksIfFileStillExists() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO() { @@ -346,7 +346,7 @@ public void close() { @Test public void testTableCleanupMultipleSnapshots() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -491,7 +491,7 @@ public void testTableCleanupMultipleSnapshots() throws IOException { @Test public void testTableCleanupMultipleMetadata() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java index 9058de0981..7c7b6af8ae 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java @@ -43,8 +43,8 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.quarkus.auth.TokenUtils; import org.junit.jupiter.api.TestInfo; @@ -106,7 +106,7 @@ private PolarisPrincipalSecrets fetchAdminSecrets() { helper.realmContextResolver.resolveRealmContext( baseUri.toString(), "GET", "/", Map.of(REALM_PROPERTY_KEY, realm)); - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = helper.metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisContext = new PolarisCallContext( diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 5ec318693c..f2ba62add8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -62,10 +62,10 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; import org.apache.polaris.service.context.CallContextCatalogFactory; @@ -123,7 +123,7 @@ public class IcebergCatalogAdapter private final CallContextCatalogFactory catalogFactory; private final PolarisEntityManager entityManager; private final PolarisMetaStoreManager metaStoreManager; - private final PolarisMetaStoreSession session; + private final TransactionalPersistence session; private final PolarisConfigurationStore configurationStore; private final PolarisDiagnostics diagnostics; private final PolarisAuthorizer polarisAuthorizer; @@ -136,7 +136,7 @@ public IcebergCatalogAdapter( CallContextCatalogFactory catalogFactory, PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, - PolarisMetaStoreSession session, + TransactionalPersistence session, PolarisConfigurationStore configurationStore, PolarisDiagnostics diagnostics, PolarisAuthorizer polarisAuthorizer, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index 2c9b928c94..48d9a280ab 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -37,7 +37,6 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -83,8 +82,6 @@ public FileIO loadFileIO( realmEntityManagerFactory.getOrCreateEntityManager(realmContext); PolarisCredentialVendor credentialVendor = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - PolarisMetaStoreSession metaStoreSession = - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); // Get subcoped creds properties = new HashMap<>(properties); diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index ff39497247..14246692e3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -29,7 +29,7 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +60,11 @@ public CallContext resolveCallContext( .addKeyValue("headers", headers) .log("Resolving CallContext"); - PolarisMetaStoreSession metaStoreSession = + // TODO: Once we have non-transactional-database persistence stores, this should be + // pushed down for the metaStoreManagerFactory to inject Transactional-DB specific things + // (including the MetaStoreSession" into the PolarisCallContext. The non-transactional + // factories would then inject something else instead if needed. + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisContext = new PolarisCallContext(metaStoreSession, diagnostics, configurationStore, clock); diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 8bf862affd..407d514cbe 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -33,10 +33,10 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; -import org.apache.polaris.core.persistence.PolarisTreeMapStore; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.service.context.RealmContextConfiguration; @@ -69,7 +69,7 @@ protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics dia } @Override - protected PolarisMetaStoreSession createMetaStoreSession( + protected TransactionalPersistence createMetaStoreSession( @Nonnull PolarisTreeMapStore store, @Nonnull RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet, @@ -89,7 +89,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( } @Override - public synchronized Supplier getOrCreateSessionSupplier( + public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { String realmId = realmContext.getRealmIdentifier(); if (!bootstrappedRealms.contains(realmId)) { diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 0f9109b7a5..6451656280 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -40,7 +40,7 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; import org.apache.polaris.service.catalog.DefaultIcebergCatalogPrefixParser; @@ -135,7 +135,7 @@ public TestServices build() { realmEntityManagerFactory.getOrCreateEntityManager(realmContext); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); CallContext callContext = new CallContext() {