diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/hive/MetadataLocator.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/hive/MetadataLocator.java new file mode 100644 index 000000000000..c09212feb363 --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/hive/MetadataLocator.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.hive; + +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.GetTableProjectionsSpecBuilder; +import org.apache.iceberg.BaseMetastoreTableOperations; +import org.apache.iceberg.ClientPool; +import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.thrift.TException; + +/** + * Fetches the location of a given metadata table. + *

Since the location mutates with each transaction, this allows determining if a cached version of the + * table is the latest known in the HMS database.

+ */ +public class MetadataLocator { + private static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(MetadataLocator.class); + private static final GetProjectionsSpec PARAM_SPEC = + new GetTableProjectionsSpecBuilder() + .includeParameters() // only fetches table.parameters + .build(); + private final HiveCatalog catalog; + + public MetadataLocator(HiveCatalog catalog) { + this.catalog = catalog; + } + + public HiveCatalog getCatalog() { + return catalog; + } + + /** + * Returns the location of the metadata table identified by the given identifier, or null if the table is + * not a metadata table. + *

This uses the Thrift API to fetch the table parameters, which is more efficient than fetching the entire table object.

+ * @param identifier the identifier of the metadata table to fetch the location for + * @return the location of the metadata table, or null if the table does not exist or is not a metadata table + * @throws NoSuchTableException if the table does not exist + */ + public String getLocation(TableIdentifier identifier) { + final ClientPool clients = catalog.clientPool(); + final String catName = catalog.name(); + final TableIdentifier baseTableIdentifier; + if (!catalog.isValidIdentifier(identifier)) { + if (!isValidMetadataIdentifier(identifier)) { + return null; + } else { + baseTableIdentifier = TableIdentifier.of(identifier.namespace().levels()); + } + } else { + baseTableIdentifier = identifier; + } + String database = baseTableIdentifier.namespace().level(0); + String tableName = baseTableIdentifier.name(); + try { + List tables = + clients.run(client -> client.getTables(catName, database, Collections.singletonList(tableName), PARAM_SPEC)); + if (tables != null && !tables.isEmpty()) { + Table table = tables.getFirst(); + if (table != null) { + HiveOperationsBase.validateTableIsIceberg(table, tableName); + return table.getParameters().get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP); + } + } + return null; + } catch (NoSuchObjectException e) { + // NoSuchObjectException is a TException subclass that HMS may raise for an unknown database or catalog. + LOGGER.debug("Table {} not found: {}", baseTableIdentifier, e.getMessage()); + throw new NoSuchTableException(e, "Table %s not found: %s", baseTableIdentifier, e.getMessage()); + } catch (TException e) { + LOGGER.warn("Table {} parameters fetch failed: {}", baseTableIdentifier, e.getMessage()); + throw new RuntimeException("Failed to fetch table parameters for " + baseTableIdentifier, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while fetching table parameters for " + baseTableIdentifier, e); + } + } + + private boolean isValidMetadataIdentifier(TableIdentifier identifier) { + return MetadataTableType.from(identifier.name()) != null + && catalog.isValidIdentifier(TableIdentifier.of(identifier.namespace().levels())); + } +} diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java index edb5fbd41a9b..d3ae82cb5725 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java @@ -7,24 +7,42 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.iceberg.rest; -import com.github.benmanes.caffeine.cache.Ticker; +import java.io.Closeable; +import java.lang.management.ManagementFactory; +import java.lang.ref.SoftReference; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +import javax.management.JMException; +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.iceberg.BaseMetadataTable; import org.apache.iceberg.CachingCatalog; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; @@ -33,58 +51,411 @@ import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.hive.HiveCatalog; +import org.apache.iceberg.hive.MetadataLocator; import org.apache.iceberg.view.View; import org.apache.iceberg.view.ViewBuilder; +import org.jetbrains.annotations.TestOnly; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.github.benmanes.caffeine.cache.Ticker; /** * Class that wraps an Iceberg Catalog to cache tables. */ -public class HMSCachingCatalog extends CachingCatalog implements SupportsNamespaces, ViewCatalog { +public class HMSCachingCatalog extends CachingCatalog + implements SupportsNamespaces, ViewCatalog, HMSCachingCatalogMXBean, Closeable { + protected static final Logger LOG = LoggerFactory.getLogger(HMSCachingCatalog.class); + + @TestOnly + private static SoftReference cacheRef = new SoftReference<>(null); + + @TestOnly + @SuppressWarnings("unchecked") + public static C getLatestCache(Function extractor) { + HMSCachingCatalog cache = cacheRef.get(); + if (cache == null) { + return null; + } + return extractor == null ? (C) cache : extractor.apply(cache); + } + + @TestOnly + public HiveCatalog getCatalog() { + return hiveCatalog; + } + + // The underlying HiveCatalog instance. private final HiveCatalog hiveCatalog; - - public HMSCachingCatalog(HiveCatalog catalog, long expiration) { - super(catalog, true, expiration, Ticker.systemTicker()); + // Duplicate because CachingCatalog doesn't expose the case sensitivity of the underlying catalog, + // which is needed for canonicalizing identifiers before caching. + private final boolean caseSensitive; + // The locator. + private final MetadataLocator metadataLocator; + // An L1 small latency cache. + // This is used to cache the last cached time for each table identifier, + // so that we can skip location check for repeated access to the same table within a short period of time, + // which can significantly reduce the latency for repeated access to the same table. + private final Map l1Cache; + // The TTL for L1 cache (3s). + private final int l1Ttl; + // The L1 cache size. + private final int l1CacheSize; + + // Metrics counters. + private final AtomicLong cacheHitCount = new AtomicLong(0); + private final AtomicLong cacheMissCount = new AtomicLong(0); + private final AtomicLong cacheLoadCount = new AtomicLong(0); + private final AtomicLong cacheInvalidateCount = new AtomicLong(0); + private final AtomicLong cacheMetaLoadCount = new AtomicLong(0); + // L1 cache metrics: counted only when the L2 (Caffeine) cache already has the entry. + private final AtomicLong l1CacheHitCount = new AtomicLong(0); + private final AtomicLong l1CacheMissCount = new AtomicLong(0); + + // JMX ObjectName under which this instance is registered (may be null if registration failed). + private ObjectName jmxObjectName; + + public HMSCachingCatalog(HiveCatalog catalog, long expirationMs) { + this(catalog, expirationMs, /*caseSensitive*/ true); + } + + public HMSCachingCatalog(HiveCatalog catalog, long expirationMs, boolean caseSensitive) { + super(catalog, caseSensitive, expirationMs, Ticker.systemTicker()); this.hiveCatalog = catalog; + this.caseSensitive = caseSensitive; + this.metadataLocator = new MetadataLocator(catalog); + Configuration conf = catalog.getConf(); + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEST)) { + // Only keep a reference to the latest cache for testing purpose, so that tests can manipulate the catalog. + cacheRef = new SoftReference<>(this); + } + int l1size = conf.getInt("hms.caching.catalog.l1.cache.size", 32); + int l1ttl = conf.getInt("hms.caching.catalog.l1.cache.ttl", 3_000); + if (l1size > 0 && l1ttl > 0) { + l1Cache = Collections.synchronizedMap(new LinkedHashMap() { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > l1CacheSize; + } + }); + l1Ttl = l1ttl; + l1CacheSize = l1size; + } else { + l1Cache = Collections.emptyMap(); + l1Ttl = 0; + l1CacheSize = 0; + } + registerJmx(catalog.name()); + } + + /** + * Registers this instance as a JMX MBean. + * + * @param catalogName the catalog name, used to build the {@link ObjectName} + */ + private void registerJmx(String catalogName) { + try { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + String sanitized = catalogName == null || catalogName.isEmpty() + ? "default" + : catalogName.replaceAll("[^a-zA-Z0-9.\\\\-]", "_"); + ObjectName name = new ObjectName("org.apache.iceberg.rest:type=HMSCachingCatalog,name=" + sanitized); + if (mbs.isRegistered(name)) { + mbs.unregisterMBean(name); + } + mbs.registerMBean(this, name); + this.jmxObjectName = name; + LOG.info("Registered JMX MBean: {}", name); + } catch (JMException e) { + LOG.warn("Failed to register JMX MBean for HMSCachingCatalog", e); + } + } + + /** + * Callback when cache invalidates the entry for a given table identifier. + * + * @param tid the table identifier to invalidate + */ + protected void onCacheInvalidate(TableIdentifier tid) { + long count = cacheInvalidateCount.incrementAndGet(); + LOG.debug("Cache invalidate {}: {}", tid, count); + } + + /** + * Callback when cache loads a table for a given table identifier. + * + * @param tid the table identifier + */ + protected void onCacheLoad(TableIdentifier tid) { + long count = cacheLoadCount.incrementAndGet(); + LOG.debug("Cache load {}: {}", tid, count); + } + + /** + * Callback when cache hit for a given table identifier. + * + * @param tid the table identifier + */ + protected void onCacheHit(TableIdentifier tid) { + long count = cacheHitCount.incrementAndGet(); + LOG.debug("Cache hit {} : {}", tid, count); + } + + /** + * Callback when cache miss occurs for a given table identifier. + * + * @param tid the table identifier + */ + protected void onCacheMiss(TableIdentifier tid) { + long count = cacheMissCount.incrementAndGet(); + LOG.debug("Cache miss {}: {}", tid, count); + } + + /** + * Callback when cache loads a metadata table for a given table identifier. + * + * @param tid the table identifier + */ + protected void onCacheMetaLoad(TableIdentifier tid) { + long count = cacheMetaLoadCount.incrementAndGet(); + LOG.debug("Cache meta-load {}: {}", tid, count); + } + + /** + * Callback when an L1 cache hit occurs for a given table identifier. + * Only fired when the L2 cache also has the entry. + * + * @param tid the table identifier + */ + protected void onL1CacheHit(TableIdentifier tid) { + long count = l1CacheHitCount.incrementAndGet(); + LOG.debug("L1 cache hit {}: {}", tid, count); } + /** + * Callback when an L1 cache miss occurs for a given table identifier. + * Only fired when the L2 cache has the entry but L1 is absent or expired. + * + * @param tid the table identifier + */ + protected void onL1CacheMiss(TableIdentifier tid) { + long count = l1CacheMissCount.incrementAndGet(); + LOG.debug("L1 cache miss {}: {}", tid, count); + } + + // Getter methods for accessing metrics @Override - public Catalog.TableBuilder buildTable(TableIdentifier identifier, Schema schema) { - return hiveCatalog.buildTable(identifier, schema); + public long getCacheHitCount() { + return cacheHitCount.get(); } @Override - public void createNamespace(Namespace nmspc, Map map) { - hiveCatalog.createNamespace(nmspc, map); + public long getCacheMissCount() { + return cacheMissCount.get(); } @Override - public List listNamespaces(Namespace nmspc) throws NoSuchNamespaceException { - return hiveCatalog.listNamespaces(nmspc); + public long getCacheLoadCount() { + return cacheLoadCount.get(); } @Override - public Map loadNamespaceMetadata(Namespace nmspc) throws NoSuchNamespaceException { - return hiveCatalog.loadNamespaceMetadata(nmspc); + public long getCacheInvalidateCount() { + return cacheInvalidateCount.get(); } @Override - public boolean dropNamespace(Namespace nmspc) throws NamespaceNotEmptyException { - List tables = listTables(nmspc); + public long getCacheMetaLoadCount() { + return cacheMetaLoadCount.get(); + } + + @Override + public double getCacheHitRate() { + long hits = cacheHitCount.get(); + long total = hits + cacheMissCount.get(); + return total == 0 ? 0.0 : (double) hits / total; + } + + @Override + public long getL1CacheHitCount() { + return l1CacheHitCount.get(); + } + + @Override + public long getL1CacheMissCount() { + return l1CacheMissCount.get(); + } + + @Override + public double getL1CacheHitRate() { + long hits = l1CacheHitCount.get(); + long total = hits + l1CacheMissCount.get(); + return total == 0 ? 0.0 : (double) hits / total; + } + + @Override + public void resetCacheStats() { + cacheHitCount.set(0); + cacheMissCount.set(0); + cacheLoadCount.set(0); + cacheInvalidateCount.set(0); + cacheMetaLoadCount.set(0); + l1CacheHitCount.set(0); + l1CacheMissCount.set(0); + LOG.debug("Cache stats reset"); + } + + @Override + public void close() { + unregisterJmx(); + } + + /** + * Unregisters this instance from the platform MBeanServer. + */ + private void unregisterJmx() { + if (jmxObjectName != null) { + try { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + if (mbs.isRegistered(jmxObjectName)) { + mbs.unregisterMBean(jmxObjectName); + LOG.info("Unregistered JMX MBean: {}", jmxObjectName); + } + } catch (JMException e) { + LOG.warn("Failed to unregister JMX MBean: {}", jmxObjectName, e); + } finally { + jmxObjectName = null; + } + } + } + + @Override + public void createNamespace(Namespace namespace, Map map) { + hiveCatalog.createNamespace(namespace, map); + } + + @Override + public List listNamespaces(Namespace namespace) throws NoSuchNamespaceException { + return hiveCatalog.listNamespaces(namespace); + } + + /** + * Canonicalizes the given table identifier based on the case sensitivity of the underlying catalog. + * Copied from CachingCatalog that exposes it as private. + * @param tableIdentifier the table identifier to canonicalize + * @return the canonicalized table identifier + */ + private TableIdentifier canonicalizeIdentifier(TableIdentifier tableIdentifier) { + return this.caseSensitive ? tableIdentifier : tableIdentifier.toLowerCase(); + } + + @Override + public void invalidateTable(TableIdentifier ident) { + super.invalidateTable(ident); + l1Cache.remove(ident); + } + + @Override + public Table loadTable(final TableIdentifier identifier) { + final TableIdentifier canonicalized = canonicalizeIdentifier(identifier); + final Table cachedTable = tableCache.getIfPresent(canonicalized); + long now = System.currentTimeMillis(); + if (cachedTable != null) { + // Determine if L1 cache is valid based on the last cached time and the TTL. + // If the table is in L1 cache, we can skip the location check and return the cached table directly, + // which can significantly reduce the latency for repeated access to the same table. + Long lastCached = l1Cache.get(canonicalized); + if (lastCached != null) { + if (now - lastCached < l1Ttl) { + LOG.debug("Table {} is in L1 cache, returning cached table", canonicalized); + onL1CacheHit(canonicalized); + onCacheHit(canonicalized); + return cachedTable; + } else { + l1Cache.remove(canonicalized); + onL1CacheMiss(canonicalized); + } + } else { + onL1CacheMiss(canonicalized); + } + // If the table is no longer in L1 cache, we need to check the location. + final String location = metadataLocator.getLocation(canonicalized); + if (location == null) { + LOG.debug("Table {} has no location, returning cached table without location", canonicalized); + onCacheHit(canonicalized); + l1Cache.put(canonicalized, now); + return cachedTable; + } + String cachedLocation = + cachedTable instanceof HasTableOperations tableOps ? tableOps.operations().current().metadataFileLocation() : null; + if (location.equals(cachedLocation)) { + onCacheHit(canonicalized); + l1Cache.put(canonicalized, now); + return cachedTable; + } else { + LOG.debug("Invalidate table {}, cached {} != actual {}", canonicalized, cachedLocation, location); + // Invalidate the cached table if the location is different + invalidateTable(canonicalized); + onCacheInvalidate(canonicalized); + } + } else { + onCacheMiss(canonicalized); + } + // The following code is copied from CachingCatalog.loadTable(), but with additional handling for L1 cache and stats. + final Table table = tableCache.get(canonicalized, this::loadTableWithoutCache); + if (table instanceof BaseMetadataTable) { + // Cache underlying table: there must be a table named by the namespace (?) + TableIdentifier originTableIdentifier = TableIdentifier.of(canonicalized.namespace().levels()); + Table originTable = tableCache.get(originTableIdentifier, this::loadTableWithoutCache); + // Share TableOperations instance of origin table for all metadata tables, so that metadata + // table instances are refreshed as well when origin table instance is refreshed. + if (originTable instanceof HasTableOperations tableOps) { + TableOperations ops = tableOps.operations(); + MetadataTableType type = MetadataTableType.from(canonicalized.name()); + // Defensive: CachingCatalog doesn't perform this check + if (type != null) { + Table metadataTable = + MetadataTableUtils.createMetadataTableInstance(ops, hiveCatalog.name(), originTableIdentifier, canonicalized, type); + tableCache.put(canonicalized, metadataTable); + l1Cache.put(canonicalized, now); + onCacheMetaLoad(canonicalized); + LOG.debug("Loaded metadata table: {} for origin table: {}", canonicalized, originTableIdentifier); + // Return the metadata table instead of the original table + return metadataTable; + } + } + } + l1Cache.put(canonicalized, now); + onCacheLoad(canonicalized); + return table; + } + + private Table loadTableWithoutCache(TableIdentifier identifier) { + return hiveCatalog.loadTable(identifier); + } + + @Override + public Map loadNamespaceMetadata(Namespace namespace) throws NoSuchNamespaceException { + return hiveCatalog.loadNamespaceMetadata(namespace); + } + + @Override + public boolean dropNamespace(Namespace namespace) throws NamespaceNotEmptyException { + List tables = listTables(namespace); for (TableIdentifier ident : tables) { invalidateTable(ident); } - return hiveCatalog.dropNamespace(nmspc); + return hiveCatalog.dropNamespace(namespace); } @Override - public boolean setProperties(Namespace nmspc, Map map) throws NoSuchNamespaceException { - return hiveCatalog.setProperties(nmspc, map); + public boolean setProperties(Namespace namespace, Map map) throws NoSuchNamespaceException { + return hiveCatalog.setProperties(namespace, map); } @Override - public boolean removeProperties(Namespace nmspc, Set set) throws NoSuchNamespaceException { - return hiveCatalog.removeProperties(nmspc, set); + public boolean removeProperties(Namespace namespace, Set set) throws NoSuchNamespaceException { + return hiveCatalog.removeProperties(namespace, set); } @Override @@ -92,6 +463,11 @@ public boolean namespaceExists(Namespace namespace) { return hiveCatalog.namespaceExists(namespace); } + @Override + public Catalog.TableBuilder buildTable(TableIdentifier identifier, Schema schema) { + return hiveCatalog.buildTable(identifier, schema); + } + @Override public List listViews(Namespace namespace) { return hiveCatalog.listViews(namespace); diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalogMXBean.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalogMXBean.java new file mode 100644 index 000000000000..d72c8e103ade --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalogMXBean.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.rest; + +/** + * JMX MXBean interface for {@link HMSCachingCatalog} that exposes cache performance statistics. + *

+ * Instances are registered under the object name: + * {@code org.apache.iceberg.rest:type=HMSCachingCatalog,name=<catalogName>}. + *

+ */ +public interface HMSCachingCatalogMXBean { + + /** + * Returns the total number of cache hits (table found in cache and still valid). + * + * @return cache hit count + */ + long getCacheHitCount(); + + /** + * Returns the total number of cache misses (table not found in cache). + * + * @return cache miss count + */ + long getCacheMissCount(); + + /** + * Returns the total number of times a table was loaded from the underlying catalog and stored in cache. + * + * @return cache load count + */ + long getCacheLoadCount(); + + /** + * Returns the total number of times a cached table was invalidated because the actual metadata location differed. + * + * @return cache invalidation count + */ + long getCacheInvalidateCount(); + + /** + * Returns the total number of times a metadata (virtual) table was loaded and cached. + * + * @return cache metadata-table load count + */ + long getCacheMetaLoadCount(); + + /** + * Returns the cache hit rate as a value in the range {@code [0.0, 1.0]}. + * Returns {@code 0.0} when no lookups have been performed. + * + * @return cache hit rate + */ + double getCacheHitRate(); + + /** + * Returns the total number of L1 (short-lived in-memory) cache hits. + * An L1 hit means the table was found in the L2 cache and its L1 TTL had not yet expired, + * so the HMS metadata-location check was skipped entirely. + * + * @return L1 cache hit count + */ + long getL1CacheHitCount(); + + /** + * Returns the total number of L1 cache misses. + * An L1 miss means the table was in the L2 cache but the L1 entry was absent or expired, + * so an HMS metadata-location check was required. + * + * @return L1 cache miss count + */ + long getL1CacheMissCount(); + + /** + * Returns the L1 cache hit rate as a value in the range {@code [0.0, 1.0]}. + * This reflects how often the short-circuit L1 path is taken vs. the full HMS location check. + * Returns {@code 0.0} when no L2-cache-hit lookups have been performed. + * + * @return L1 cache hit rate + */ + double getL1CacheHitRate(); + + /** + * Resets all cache statistics counters to zero. + */ + void resetCacheStats(); +} diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java index 8a9ea142d72d..fb8645aa0dc4 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java @@ -7,14 +7,13 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.iceberg.rest; @@ -25,6 +24,7 @@ import java.io.IOException; import java.time.Clock; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import javax.servlet.http.HttpServletResponse; @@ -83,6 +83,7 @@ * Original @ RESTCatalogAdapter.java * Adaptor class to translate REST requests into {@link Catalog} API calls. */ + public class HMSCatalogAdapter implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogAdapter.class); private static final Splitter SLASH = Splitter.on('/'); @@ -400,10 +401,9 @@ private static void commitTransaction(Catalog catalog, CommitTransactionRequest for (UpdateTableRequest tableChange : request.tableChanges()) { Table table = catalog.loadTable(tableChange.identifier()); - if (table instanceof BaseTable) { - Transaction transaction = - Transactions.newTransaction( - tableChange.identifier().toString(), ((BaseTable) table).operations()); + if (table instanceof BaseTable baseTable) { + Transaction transaction = Transactions.newTransaction( + tableChange.identifier().toString(), baseTable.operations()); transactions.add(transaction); BaseTransaction.TransactionTable txTable = @@ -420,87 +420,35 @@ private static void commitTransaction(Catalog catalog, CommitTransactionRequest } @SuppressWarnings({"MethodLength", "unchecked"}) - private T handleRequest( - Route route, Map vars, Object body) { - switch (route) { - case CONFIG: - return (T) config(); - - case LIST_NAMESPACES: - return (T) listNamespaces(vars); - - case CREATE_NAMESPACE: - return (T) createNamespace(body); - - case NAMESPACE_EXISTS: - return (T) namespaceExists(vars); - - case LOAD_NAMESPACE: - return (T) loadNamespace(vars); - - case DROP_NAMESPACE: - return (T) dropNamespace(vars); - - case UPDATE_NAMESPACE: - return (T) updateNamespace(vars, body); - - case LIST_TABLES: - return (T) listTables(vars); - - case CREATE_TABLE: - return (T) createTable(vars, body); - - case DROP_TABLE: - return (T) dropTable(vars); - - case TABLE_EXISTS: - return (T) tableExists(vars); - - case LOAD_TABLE: - return (T) loadTable(vars); - - case REGISTER_TABLE: - return (T) registerTable(vars, body); - - case UPDATE_TABLE: - return (T) updateTable(vars, body); - - case RENAME_TABLE: - return (T) renameTable(body); - - case REPORT_METRICS: - return (T) reportMetrics(vars, body); - - case COMMIT_TRANSACTION: - return (T) commitTransaction(body); - - case LIST_VIEWS: - return (T) listViews(vars); - - case CREATE_VIEW: - return (T) createView(vars, body); - - case VIEW_EXISTS: - return (T) viewExists(vars); - - case LOAD_VIEW: - return (T) loadView(vars); - - case UPDATE_VIEW: - return (T) updateView(vars, body); - - case RENAME_VIEW: - return (T) renameView(body); - - case DROP_VIEW: - return (T) dropView(vars); - - default: - } - return null; + private T handleRequest(Route route, Map vars, Object body) { + return switch (route) { + case CONFIG -> (T) config(); + case LIST_NAMESPACES -> (T) listNamespaces(vars); + case CREATE_NAMESPACE -> (T) createNamespace(body); + case NAMESPACE_EXISTS -> (T) namespaceExists(vars); + case LOAD_NAMESPACE -> (T) loadNamespace(vars); + case DROP_NAMESPACE -> (T) dropNamespace(vars); + case UPDATE_NAMESPACE -> (T) updateNamespace(vars, body); + case LIST_TABLES -> (T) listTables(vars); + case CREATE_TABLE -> (T) createTable(vars, body); + case DROP_TABLE -> (T) dropTable(vars); + case TABLE_EXISTS -> (T) tableExists(vars); + case LOAD_TABLE -> (T) loadTable(vars); + case REGISTER_TABLE -> (T) registerTable(vars, body); + case UPDATE_TABLE -> (T) updateTable(vars, body); + case RENAME_TABLE -> (T) renameTable(body); + case REPORT_METRICS -> (T) reportMetrics(vars, body); + case COMMIT_TRANSACTION -> (T) commitTransaction(body); + case LIST_VIEWS -> (T) listViews(vars); + case CREATE_VIEW -> (T) createView(vars, body); + case VIEW_EXISTS -> (T) viewExists(vars); + case LOAD_VIEW -> (T) loadView(vars); + case UPDATE_VIEW -> (T) updateView(vars, body); + case RENAME_VIEW -> (T) renameView(body); + case DROP_VIEW -> (T) dropView(vars); + }; } - T execute( HTTPMethod method, String path, diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java index d7bd3251a0c7..b8a0e6f6ea2b 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java @@ -7,15 +7,15 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.apache.iceberg.rest; import java.lang.reflect.InvocationTargetException; diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java index 21b155d65d8d..042a4ae2b7e3 100644 --- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java +++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java @@ -7,14 +7,13 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.iceberg.rest; @@ -29,6 +28,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.rest.HMSCatalogAdapter.Route; import org.apache.iceberg.rest.HTTPRequest.HTTPMethod; +import org.apache.iceberg.exceptions.RESTException; import org.apache.iceberg.rest.responses.ErrorResponse; import org.apache.iceberg.util.Pair; import org.slf4j.Logger; @@ -80,6 +80,11 @@ protected void service(HttpServletRequest request, HttpServletResponse response) if (responseBody != null) { RESTObjectMapper.mapper().writeValue(response.getWriter(), responseBody); } + } catch (RESTException e) { + // A RESTException is thrown by HMSCatalogAdapter.execute() after the error handler has + // already written the correct HTTP status and body to the response (e.g. 404, 403). + // It is not an unexpected server failure, so log at DEBUG to avoid flooding the console. + LOG.debug("REST request resulted in a client error (already handled): {}", e.getMessage()); } catch (RuntimeException | IOException e) { LOG.error("Error processing REST request", e); response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCachingCatalogStats.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCachingCatalogStats.java new file mode 100644 index 000000000000..33317f0b30ab --- /dev/null +++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCachingCatalogStats.java @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest; + +import java.lang.management.ManagementFactory; +import java.util.Set; + +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hive.HiveCatalog; +import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; +import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Integration tests that verify the {@link HMSCachingCatalog} cache-statistics counters + * (hit, miss, load, invalidate, l1-hit, l1-miss, and their rates) are updated correctly + * and exposed accurately via the JMX MBean registered under + * {@code org.apache.iceberg.rest:type=HMSCachingCatalog,name=*}. + * + *

The server is started with {@link AuthType#NONE} so the tests focus purely on + * caching behaviour without any authentication noise. + */ +@Category(MetastoreCheckinTest.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestHMSCachingCatalogStats { + + /** 5 minutes expressed in milliseconds – the value injected into {@code ICEBERG_CATALOG_CACHE_EXPIRY}. */ + private static final long CACHE_EXPIRY_MS = 5 * 60 * 1_000L; + + @RegisterExtension + private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION = HiveRESTCatalogServerExtension.builder(AuthType.NONE) + // Without a positive expiry the HMSCatalogFactory skips HMSCachingCatalog entirely. + .configure(MetastoreConf.ConfVars.ICEBERG_CATALOG_CACHE_EXPIRY.getVarname(), String.valueOf(CACHE_EXPIRY_MS)) + .configure("hive.in.test", "true").build(); + + private RESTCatalog catalog; + private HiveCatalog serverCatalog; + /** The server-side {@link HMSCachingCatalog} instance; used to invalidate entries directly. */ + private HMSCachingCatalog serverCachingCatalog; + /** The platform {@link MBeanServer} used for all JMX-based assertions. */ + private MBeanServer mbs; + /** Resolved once in {@link #setupAll()} and reused across every test. */ + private ObjectName jmxObjectName; + + @BeforeAll + void setupAll() throws Exception { + catalog = RCKUtils.initCatalogClient(java.util.Map.of("uri", REST_CATALOG_EXTENSION.getRestEndpoint())); + serverCachingCatalog = HMSCachingCatalog.getLatestCache(null); + Assertions.assertNotNull(serverCachingCatalog, "Expected HMSCachingCatalog to be initialized"); + serverCatalog = serverCachingCatalog.getCatalog(); + + // Resolve the JMX ObjectName registered by HMSCachingCatalog. We use a wildcard + // so the test is independent of the exact catalog name. + mbs = ManagementFactory.getPlatformMBeanServer(); + Set names = mbs.queryNames( + new ObjectName("org.apache.iceberg.rest:type=HMSCachingCatalog,*"), null); + Assertions.assertFalse(names.isEmpty(), + "HMSCachingCatalog MBean must be registered in the platform MBeanServer"); + jmxObjectName = names.iterator().next(); + } + + /** Remove any namespace/table created by the test so each run starts clean. */ + @AfterEach + void cleanup() { + RCKUtils.purgeCatalogTestEntries(catalog); + } + + // --------------------------------------------------------------------------- + // helpers + // --------------------------------------------------------------------------- + + /** + * Reads a single JMX attribute from the {@link HMSCachingCatalogMXBean}. + * + * @param attribute the attribute name as declared in {@link HMSCachingCatalogMXBean} + * (e.g. {@code "CacheHitCount"}) + * @return the attribute value + */ + private Object getJmxAttribute(String attribute) throws Exception { + return mbs.getAttribute(jmxObjectName, attribute); + } + + /** + * Convenience wrapper that reads a {@code long} JMX attribute. + */ + private long jmxLong(String attribute) throws Exception { + return (long) getJmxAttribute(attribute); + } + + /** + * Convenience wrapper that reads a {@code double} JMX attribute. + */ + private double jmxDouble(String attribute) throws Exception { + return (double) getJmxAttribute(attribute); + } + + /** + * Invokes a void JMX operation on the {@link HMSCachingCatalogMXBean}. + * + * @param operationName the operation name (e.g. {@code "resetCacheStats"}) + */ + private void invokeJmxOperation(String operationName) throws Exception { + mbs.invoke(jmxObjectName, operationName, new Object[0], new String[0]); + } + + // --------------------------------------------------------------------------- + // tests + // --------------------------------------------------------------------------- + + /** + * Verifies that the {@link HMSCachingCatalog} correctly tracks cache hits, misses, + * loads, invalidations, L1 hits, and L1 misses via JMX. + * + *

Strategy: + *

    + *
  1. Snapshot JMX baseline counters before any operations so the test is isolated + * from cumulative state left by previous tests.
    2. + *
  2. Create a namespace and a table.
    3. + *
  3. First {@code loadTable} call → cache miss + actual load.
    4. + *
  4. Second and third rapid {@code loadTable} calls → L1 cache hits (TTL still valid).
    5. + *
  5. Mutate the table to advance its metadata location in HMS.
    6. + *
  6. Wait for the L1 TTL to expire, then reload → L1 miss + invalidation + reload.
    7. + *
  7. Assert JMX counter deltas match expectations.
    8. + *
+ */ + @Test + void testCacheCountersAreUpdated() throws Exception { + // -- JMX baseline ----------------------------------------------------------- + long baseHit = jmxLong("CacheHitCount"); + long baseMiss = jmxLong("CacheMissCount"); + long baseLoad = jmxLong("CacheLoadCount"); + long baseL1Hit = jmxLong("L1CacheHitCount"); + + // -- exercise the cache ----------------------------------------------------- + var db = Namespace.of("caching_stats_test_db"); + var tableId = TableIdentifier.of(db, "caching_stats_test_table"); + + catalog.createNamespace(db); + catalog.createTable(tableId, new Schema()); + + // First load → cache miss + load + catalog.loadTable(tableId); + // Second load → L1 hit (within TTL, HMS location check skipped) + catalog.loadTable(tableId); + // Third load → L1 hit + catalog.loadTable(tableId); + + // Mutate the table by appending a data file – this creates a new snapshot + // which advances METADATA_LOCATION in HMS, so the next loadTable call through + // the caching catalog will detect the stale cached location and invalidate it. + Table table = serverCatalog.loadTable(tableId); + DataFile dataFile = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(table.location() + "/data/fake-0.parquet") + .withFileSizeInBytes(1024).withRecordCount(1).build(); + table.newAppend().appendFile(dataFile).commit(); + + long baseInvalidate = jmxLong("CacheInvalidateCount"); + // The L1 cache has a 3-second default TTL; wait for entries to expire. + Thread.sleep(3_000); + // Fourth load → L1 miss + cache invalidation + reload + catalog.loadTable(tableId); + + // -- JMX assertions --------------------------------------------------------- + long deltaHit = jmxLong("CacheHitCount") - baseHit; + long deltaMiss = jmxLong("CacheMissCount") - baseMiss; + long deltaLoad = jmxLong("CacheLoadCount") - baseLoad; + long deltaInvalidate = jmxLong("CacheInvalidateCount") - baseInvalidate; + long deltaL1Hit = jmxLong("L1CacheHitCount") - baseL1Hit; + long deltaL1Miss = jmxLong("L1CacheMissCount"); // absolute value is fine for L1 miss + + Assertions.assertTrue(deltaMiss >= 1, + "Expected at least 1 cache miss (first loadTable), but delta was: " + deltaMiss); + Assertions.assertTrue(deltaLoad >= 2, + "Expected at least 2 cache loads (initial + post-invalidation reload), but delta was: " + deltaLoad); + Assertions.assertTrue(deltaHit >= 2, + "Expected at least 2 cache hits (second + third loadTable), but delta was: " + deltaHit); + Assertions.assertTrue(deltaInvalidate >= 1, + "Expected at least 1 cache invalidation (metadata location changed), but delta was: " + deltaInvalidate); + + // L1 hits: the 2nd and 3rd loadTable calls should have been served by L1. + Assertions.assertTrue(deltaL1Hit >= 2, + "Expected at least 2 L1 cache hits (rapid successive loads within TTL), but delta was: " + deltaL1Hit); + // L1 miss: at least the fourth load (after TTL expiry) must have missed L1. + Assertions.assertTrue(deltaL1Miss >= 1, + "Expected at least 1 L1 cache miss (after TTL expiry), but was: " + deltaL1Miss); + + // Rate attributes must be valid ratios in [0.0, 1.0]. + double hitRate = jmxDouble("CacheHitRate"); + Assertions.assertTrue(hitRate > 0.0 && hitRate <= 1.0, + "CacheHitRate must be in (0.0, 1.0] but was: " + hitRate); + + double l1HitRate = jmxDouble("L1CacheHitRate"); + Assertions.assertTrue(l1HitRate > 0.0 && l1HitRate <= 1.0, + "L1CacheHitRate must be in (0.0, 1.0] but was: " + l1HitRate); + } + + /** + * Verifies that the {@code resetCacheStats} JMX operation zeroes all counters. + * + *

Strategy: + *

    + *
  1. Perform some cache operations to ensure all counters are non-zero.
    2. + *
  2. Invoke {@code resetCacheStats()} via JMX.
    3. + *
  3. Assert that every JMX counter attribute reads {@code 0} / {@code 0.0}.
    4. + *
+ */ + @Test + void testJmxResetCacheStats() throws Exception { + // -- warm up counters ------------------------------------------------------- + var db = Namespace.of("jmx_reset_test_db"); + var tableId = TableIdentifier.of(db, "jmx_reset_test_table"); + catalog.createNamespace(db); + catalog.createTable(tableId, new Schema()); + catalog.loadTable(tableId); // miss + load + catalog.loadTable(tableId); // hit (L1 hit on the fast path) + + // Sanity: at least one counter must be non-zero before the reset. + Assertions.assertTrue(jmxLong("CacheHitCount") + jmxLong("CacheMissCount") > 0, + "At least one counter must be non-zero before reset"); + + // -- invoke the reset operation via JMX ------------------------------------- + invokeJmxOperation("resetCacheStats"); + + // -- assertions post-reset -------------------------------------------------- + Assertions.assertEquals(0L, jmxLong("CacheHitCount"), "CacheHitCount must be 0 after reset"); + Assertions.assertEquals(0L, jmxLong("CacheMissCount"), "CacheMissCount must be 0 after reset"); + Assertions.assertEquals(0L, jmxLong("CacheLoadCount"), "CacheLoadCount must be 0 after reset"); + Assertions.assertEquals(0L, jmxLong("CacheInvalidateCount"), "CacheInvalidateCount must be 0 after reset"); + Assertions.assertEquals(0L, jmxLong("CacheMetaLoadCount"), "CacheMetaLoadCount must be 0 after reset"); + Assertions.assertEquals(0L, jmxLong("L1CacheHitCount"), "L1CacheHitCount must be 0 after reset"); + Assertions.assertEquals(0L, jmxLong("L1CacheMissCount"), "L1CacheMissCount must be 0 after reset"); + Assertions.assertEquals(0.0, jmxDouble("CacheHitRate"), 1e-9, "CacheHitRate must be 0.0 after reset"); + Assertions.assertEquals(0.0, jmxDouble("L1CacheHitRate"), 1e-9, "L1CacheHitRate must be 0.0 after reset"); + + // -- verify rate calculation still works correctly after reset -------------- + // resetCacheStats() zeroes counters but does NOT evict the L2/L1 cache, so the + // table is still cached. Invalidate it directly on the server-side HMSCachingCatalog + // so the first post-reset load is a genuine cold miss rather than an L1/L2 hit. + // NOTE: catalog.invalidateTable() only clears the REST *client* state and does not + // reach the server-side cache. + serverCachingCatalog.invalidateTable(tableId); + + // First load after reset: cache miss + load (L1 cold, L2 cold). + catalog.loadTable(tableId); + // Second load: L1 hit (within TTL). + catalog.loadTable(tableId); + // Third load: L1 hit (within TTL). + catalog.loadTable(tableId); + + // CacheHitRate: 2 hits out of 3 total accesses → ≈ 0.667 + double hitRateAfterReset = jmxDouble("CacheHitRate"); + Assertions.assertTrue(hitRateAfterReset > 0.0 && hitRateAfterReset <= 1.0, + "CacheHitRate must be in (0.0, 1.0] after post-reset operations, but was: " + hitRateAfterReset); + + // Underlying counters must reflect the just-performed operations. + Assertions.assertTrue(jmxLong("CacheHitCount") >= 2, + "CacheHitCount must be >= 2 after two rapid re-loads post-reset"); + Assertions.assertTrue(jmxLong("CacheMissCount") >= 1, + "CacheMissCount must be >= 1 after the first cold load post-reset"); + + // L1CacheHitRate: the 2nd and 3rd loads should have been served by L1. + double l1HitRateAfterReset = jmxDouble("L1CacheHitRate"); + Assertions.assertTrue(l1HitRateAfterReset > 0.0 && l1HitRateAfterReset <= 1.0, + "L1CacheHitRate must be in (0.0, 1.0] after post-reset L1 hits, but was: " + l1HitRateAfterReset); + + Assertions.assertTrue(jmxLong("L1CacheHitCount") >= 2, + "L1CacheHitCount must be >= 2 after two rapid re-loads within TTL post-reset"); + } +}