[#19304] YSQL: Avoid reading catalog version from a master in case ta…

…bles are prefetched without response cache

Summary:
On connection initialization (for authorization purposes ) YSQL makes read from multiple system tables by prefetching them at once. This prefetching doesn't use response cache. But the call of the `YBCStartSysTablePrefetching` function requires catalog version even in case the `YB_YQL_PREFETCHER_NO_CACHE` mode is used. Fetching catalog version requires RPC to a master.

Solution is to remove the `YB_YQL_PREFETCHER_NO_CACHE` enum value and introduce new function `YBCStartSysTablePrefetchingNoCache` which accepts no arguments.

Jira: DB-8115

Test Plan:
Jenkins

```
./yb_build.sh --gtest_filter PgCatalogPerfTest.*
```

Reviewers: jason, myang

Reviewed By: myang

Subscribers: yql

Differential Revision: https://phorge.dev.yugabyte.com/D28826

src/postgres/src/backend/utils/cache/relcache.c

@@ -2513,20 +2513,25 @@ typedef struct YbRunWithPrefetcherContext
 	bool is_using_response_cache;
 } YbRunWithPrefetcherContext;
 
+typedef struct YbPrefetcherStarterFunctor
+{
+   bool (*call)(struct YbPrefetcherStarterFunctor* );
+} YbPrefetcherStarterFunctor;
+
 static YBCStatus
 YbRunWithPrefetcherImpl(
-	YBCPgLastKnownCatalogVersionInfo catalog_version,
-	YBCPgSysTablePrefetcherCacheMode cache_mode,
-	YBCStatus (*func)(YbRunWithPrefetcherContext* ctx),
+	YbPrefetcherStarterFunctor *prefetcher_starter,
+	YBCStatus (*func)(YbRunWithPrefetcherContext *ctx),
 	bool keep_prefetcher)
 {
 	YBCPgResetCatalogReadTime();
-	YBCStartSysTablePrefetching(catalog_version, cache_mode);
+	const bool is_using_response_cache =
+		prefetcher_starter->call(prefetcher_starter);
 	YBCStatus result = NULL;
 	PG_TRY();
 	{
 		YbRunWithPrefetcherContext ctx = {};
-		ctx.is_using_response_cache = cache_mode != YB_YQL_PREFETCHER_NO_CACHE;
+		ctx.is_using_response_cache = is_using_response_cache;
 		result = func(&ctx);
 	}
 	PG_CATCH();
@@ -2547,37 +2552,76 @@ YbRunWithPrefetcherImpl(
 	return result;
 }
 
+typedef struct YbPrefetcherStarterWithCache {
+	/* YbPrefetcherStarterFunctor have to be the first field due to cast */
+	YbPrefetcherStarterFunctor functor;
+	const YBCPgLastKnownCatalogVersionInfo *version;
+	YBCPgSysTablePrefetcherCacheMode mode;
+} YbPrefetcherStarterWithCache;
+
+static bool
+YbPrefetcherStarterWithCacheCall(YbPrefetcherStarterFunctor *functor) {
+	const YbPrefetcherStarterWithCache *this =
+		(const YbPrefetcherStarterWithCache *)functor;
+	YBCStartSysTablePrefetching(*this->version, this->mode);
+	return true;
+}
+
+static YbPrefetcherStarterWithCache
+MakeStarterWithCache(YBCPgSysTablePrefetcherCacheMode mode,
+					 const YBCPgLastKnownCatalogVersionInfo *version)
+{
+	return (YbPrefetcherStarterWithCache){
+		.functor = {.call = &YbPrefetcherStarterWithCacheCall},
+		.version = version,
+		.mode = mode
+	};
+}
+
+static bool
+YbPrefetcherStarterNoCacheCall(YbPrefetcherStarterFunctor *functor)
+{
+	YBCStartSysTablePrefetchingNoCache();
+	return false;
+}
+
 static void
 YbRunWithPrefetcher(
-	YBCStatus (*func)(YbRunWithPrefetcherContext*),
-	bool keep_prefetcher)
-{
-	static const YBCPgSysTablePrefetcherCacheMode cache_modes[] =
-	{
-		YB_YQL_PREFETCHER_TRUST_CACHE,
-		YB_YQL_PREFETCHER_RENEW_CACHE_SOFT,
-		YB_YQL_PREFETCHER_RENEW_CACHE_HARD,
-		YB_YQL_PREFETCHER_NO_CACHE
+	YBCStatus (*func)(YbRunWithPrefetcherContext *), bool keep_prefetcher)
+{
+	YBCPgLastKnownCatalogVersionInfo catalog_version = {};
+	YbPrefetcherStarterWithCache trust_cache = MakeStarterWithCache(
+		YB_YQL_PREFETCHER_TRUST_CACHE, &catalog_version);
+	YbPrefetcherStarterWithCache renew_soft = MakeStarterWithCache(
+		YB_YQL_PREFETCHER_RENEW_CACHE_SOFT, &catalog_version);
+	YbPrefetcherStarterWithCache renew_hard = MakeStarterWithCache(
+		YB_YQL_PREFETCHER_RENEW_CACHE_HARD, &catalog_version);
+	YbPrefetcherStarterFunctor no_cache =
+		{.call = &YbPrefetcherStarterNoCacheCall};
+
+	YbPrefetcherStarterFunctor *prefetcher_starters[] = {
+		&trust_cache.functor,
+		&renew_soft.functor,
+		&renew_hard.functor,
+		&no_cache
 	};
 
-	static const size_t kCacheModesCount = lengthof(cache_modes);
-	const YBCPgLastKnownCatalogVersionInfo catalog_version =
-		YbGetCatalogCacheVersionForTablePrefetching();
+	static const size_t kStartersCount = lengthof(prefetcher_starters);
 
-	size_t cache_mode_idx =
-		YBCIsInitDbModeEnvVarSet() ||
-		!*YBCGetGFlags()->ysql_enable_read_request_caching
-			? (kCacheModesCount - 1) : 0;
+	size_t starter_idx = kStartersCount - 1;
+	if (!YBCIsInitDbModeEnvVarSet() &&
+		*YBCGetGFlags()->ysql_enable_read_request_caching)
+	{
+		starter_idx = 0;
+		catalog_version = YbGetCatalogCacheVersionForTablePrefetching();
+	}
 	for (;;)
 	{
 		YBCStatus status = YbRunWithPrefetcherImpl(
-			catalog_version,
-			cache_modes[cache_mode_idx],
-			func,
-			keep_prefetcher);
+			prefetcher_starters[starter_idx], func, keep_prefetcher);
 		if (!status)
 			break;
-		if (++cache_mode_idx == kCacheModesCount ||
+		if (++starter_idx == kStartersCount ||
 			!YBCStatusIsSnapshotTooOld(status))
 		{
 			HandleYBStatus(status);
@@ -2925,7 +2969,7 @@ YbPrefetchRequiredDataImpl(
 	YbRunWithPrefetcherContext* ctx,
 	bool preload_rel_cache)
 {
-	YbTablePrefetcherState* prefetcher = &ctx->prefetcher;
+	YbTablePrefetcherState *prefetcher = &ctx->prefetcher;
 
 	YBCStatus status = NULL;
 	if (preload_rel_cache)
@@ -2954,13 +2998,13 @@ YbPrefetchRequiredDataImpl(
 }
 
 static YBCStatus
-YbPrefetchRequiredDataWithoutRelCache(YbRunWithPrefetcherContext* ctx)
+YbPrefetchRequiredDataWithoutRelCache(YbRunWithPrefetcherContext *ctx)
 {
 	return YbPrefetchRequiredDataImpl(ctx, false /* preload_rel_cache */);
 }
 
 static YBCStatus
-YbPrefetchRequiredDataWithRelCache(YbRunWithPrefetcherContext* ctx)
+YbPrefetchRequiredDataWithRelCache(YbRunWithPrefetcherContext *ctx)
 {
 	return YbPrefetchRequiredDataImpl(ctx, true /* preload_rel_cache */);
 }

src/postgres/src/backend/utils/init/postinit.c

@@ -693,12 +693,14 @@ InitPostgresImpl(const char *in_dbname, Oid dboid, const char *username,
 										YbRoleProfileRelationId,
 										&YbLoginProfileCatalogsExist));
 
-		const YBCPgLastKnownCatalogVersionInfo catalog_version =
-			YbGetCatalogCacheVersionForTablePrefetching();
-
+		/* TODO (dmitry): Next call of the YBIsDBCatalogVersionMode function is
+		 * kind of a hack and must be removed. This function is called before
+		 * starting prefetching because for now switching into DB catalog
+		 * version mode is impossible in case prefething is started.
+		 */
+		YBIsDBCatalogVersionMode();
 		YBCPgResetCatalogReadTime();
-		YBCStartSysTablePrefetching(
-			catalog_version, YB_YQL_PREFETCHER_NO_CACHE);
+		YBCStartSysTablePrefetchingNoCache();
 		YbRegisterSysTableForPrefetching(AuthIdRelationId);   // pg_authid
 		YbRegisterSysTableForPrefetching(DatabaseRelationId); // pg_database
 

src/yb/yql/pggate/pg_sys_table_prefetcher.cc

@@ -295,26 +295,24 @@ class VersionInfoWriter {
   return result;
 }
 
-[[nodiscard]] PgSession::CacheOptions BuildCacheOptions(
-    yb::ThreadSafeArena* arena, const ReadHybridTime& catalog_read_time,
-    const std::vector<OperationInfo>& ops, const PrefetcherOptions& options) {
-  std::optional<uint32_t> threshold_ms;
-  switch(options.cache_mode) {
-    case PrefetchingCacheMode::NO_CACHE:
-      DCHECK(false);
-      break;
+[[nodiscard]] std::optional<uint32_t> GetCacheLifetimeThreshold(PrefetchingCacheMode mode) {
+  switch(mode) {
     case PrefetchingCacheMode::TRUST_CACHE:
-      break;
+      return std::nullopt;
     case PrefetchingCacheMode::RENEW_CACHE_SOFT:
-      threshold_ms = FLAGS_pg_cache_response_renew_soft_lifetime_limit_ms;
-      break;
+      return FLAGS_pg_cache_response_renew_soft_lifetime_limit_ms;
     case PrefetchingCacheMode::RENEW_CACHE_HARD:
-      threshold_ms = 0;
-      break;
+      return 0;
   }
+  FATAL_INVALID_ENUM_VALUE(PrefetchingCacheMode, mode);
+}
+
+[[nodiscard]] PgSession::CacheOptions BuildCacheOptions(
+    yb::ThreadSafeArena* arena, const ReadHybridTime& catalog_read_time,
+    const std::vector<OperationInfo>& ops, const PrefetcherOptions::CachingInfo& caching_info) {
   return {
-      .key = BuildCacheKey(arena, catalog_read_time, ops, options.version_info),
-      .lifetime_threshold_ms = threshold_ms
+      .key = BuildCacheKey(arena, catalog_read_time, ops, caching_info.version_info),
+      .lifetime_threshold_ms = GetCacheLifetimeThreshold(caching_info.mode)
   };
 }
 
@@ -334,11 +332,11 @@ auto MakeGenerator(const std::vector<OperationInfo>& ops) {
 Result<rpc::CallResponsePtr> Run(
     yb::ThreadSafeArena* arena, PgSession* session,
     const std::vector<OperationInfo>& ops, const PrefetcherOptions& options) {
-  auto result = options.cache_mode == PrefetchingCacheMode::NO_CACHE
-    ? VERIFY_RESULT(session->RunAsync(make_lw_function(MakeGenerator(ops)), HybridTime()))
-    : VERIFY_RESULT(session->RunAsync(
+  auto result = VERIFY_RESULT(options.caching_info
+      ? session->RunAsync(
           make_lw_function(MakeGenerator(ops)),
-          BuildCacheOptions(arena, session->catalog_read_time(), ops, options)));
+          BuildCacheOptions(arena, session->catalog_read_time(), ops, *options.caching_info))
+      : session->RunAsync(make_lw_function(MakeGenerator(ops)), HybridTime()));
   return VERIFY_RESULT(result.Get()).response;
 }
 
@@ -465,8 +463,12 @@ std::string PrefetcherOptions::VersionInfo::ToString() const {
   return YB_STRUCT_TO_STRING(version, is_db_catalog_version_mode);
 }
 
+std::string PrefetcherOptions::CachingInfo::ToString() const {
+  return YB_STRUCT_TO_STRING(version_info, mode);
+}
+
 std::string PrefetcherOptions::ToString() const {
-  return YB_STRUCT_TO_STRING(version_info, cache_mode, fetch_row_limit);
+  return YB_STRUCT_TO_STRING(caching_info, fetch_row_limit);
 }
 
 class PgSysTablePrefetcher::Impl {

src/yb/yql/pggate/pg_sys_table_prefetcher.h

@@ -14,6 +14,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include <string>
 
 #include <boost/container/small_vector.hpp>
@@ -32,7 +33,7 @@ struct PgObjectId;
 namespace pggate {
 class PgSession;
 
-YB_DEFINE_ENUM(PrefetchingCacheMode, (NO_CACHE)(TRUST_CACHE)(RENEW_CACHE_SOFT)(RENEW_CACHE_HARD));
+YB_DEFINE_ENUM(PrefetchingCacheMode, (TRUST_CACHE)(RENEW_CACHE_SOFT)(RENEW_CACHE_HARD));
 
 using PrefetchedDataHolder =
     std::shared_ptr<const boost::container::small_vector<rpc::SidecarHolder, 8>>;
@@ -45,8 +46,14 @@ struct PrefetcherOptions {
     std::string ToString() const;
   };
 
-  VersionInfo version_info;
-  PrefetchingCacheMode cache_mode;
+  struct CachingInfo {
+    VersionInfo version_info;
+    PrefetchingCacheMode mode;
+
+    std::string ToString() const;
+  };
+
+  std::optional<CachingInfo> caching_info;
   uint64_t fetch_row_limit;
 
   std::string ToString() const;

src/yb/yql/pggate/ybc_pg_typedefs.h

@@ -506,8 +506,7 @@ static const int32_t kYBCMaxNumDbCatalogVersions = 10000;
 typedef enum PgSysTablePrefetcherCacheMode {
   YB_YQL_PREFETCHER_TRUST_CACHE,
   YB_YQL_PREFETCHER_RENEW_CACHE_SOFT,
-  YB_YQL_PREFETCHER_RENEW_CACHE_HARD,
-  YB_YQL_PREFETCHER_NO_CACHE
+  YB_YQL_PREFETCHER_RENEW_CACHE_HARD
 } YBCPgSysTablePrefetcherCacheMode;
 
 typedef struct PgLastKnownCatalogVersionInfo {

src/yb/yql/pggate/ybc_pggate.cc

@@ -280,6 +280,24 @@ Status GetSplitPoints(YBCPgTableDesc table_desc,
   return Status::OK();
 }
 
+void YBCStartSysTablePrefetchingImpl(std::optional<PrefetcherOptions::CachingInfo> caching_info) {
+  pgapi->StartSysTablePrefetching({caching_info, implicit_cast<uint64_t>(yb_fetch_row_limit)});
+}
+
+PrefetchingCacheMode YBCMapPrefetcherCacheMode(YBCPgSysTablePrefetcherCacheMode mode) {
+  switch (mode) {
+    case YB_YQL_PREFETCHER_TRUST_CACHE:
+      return PrefetchingCacheMode::TRUST_CACHE;
+    case YB_YQL_PREFETCHER_RENEW_CACHE_SOFT:
+      return PrefetchingCacheMode::RENEW_CACHE_SOFT;
+    case YB_YQL_PREFETCHER_RENEW_CACHE_HARD:
+      LOG(DFATAL) << "Emergency fallback prefetching cache mode is used";
+      return PrefetchingCacheMode::RENEW_CACHE_HARD;
+  }
+  LOG(DFATAL) << "Unexpected PgSysTablePrefetcherCacheMode value " << mode;
+  return PrefetchingCacheMode::RENEW_CACHE_HARD;
+}
+
 } // namespace
 
 //--------------------------------------------------------------------------------------------------
@@ -1659,28 +1677,16 @@ void* YBCPgGetThreadLocalErrStatus() {
   return PgGetThreadLocalErrStatus();
 }
 
+void YBCStartSysTablePrefetchingNoCache() {
+  YBCStartSysTablePrefetchingImpl(std::nullopt);
+}
+
 void YBCStartSysTablePrefetching(
-  YBCPgLastKnownCatalogVersionInfo version_info,
-  YBCPgSysTablePrefetcherCacheMode cache_mode) {
-  PrefetchingCacheMode mode = PrefetchingCacheMode::NO_CACHE;
-  switch (cache_mode) {
-    case YB_YQL_PREFETCHER_TRUST_CACHE:
-      mode = PrefetchingCacheMode::TRUST_CACHE;
-      break;
-    case YB_YQL_PREFETCHER_RENEW_CACHE_SOFT:
-      mode = PrefetchingCacheMode::RENEW_CACHE_SOFT;
-      break;
-    case YB_YQL_PREFETCHER_RENEW_CACHE_HARD:
-      LOG(DFATAL) << "Emergency fallback prefetching cache mode is used";
-      mode = PrefetchingCacheMode::RENEW_CACHE_HARD;
-      break;
-    default:
-      break;
-  }
-  pgapi->StartSysTablePrefetching(PrefetcherOptions{
+    YBCPgLastKnownCatalogVersionInfo version_info,
+    YBCPgSysTablePrefetcherCacheMode cache_mode) {
+  YBCStartSysTablePrefetchingImpl(PrefetcherOptions::CachingInfo{
       {version_info.version, version_info.is_db_catalog_version_mode},
-      mode,
-      implicit_cast<uint64_t>(yb_fetch_row_limit)});
+      YBCMapPrefetcherCacheMode(cache_mode)});
 }
 
 void YBCStopSysTablePrefetching() {

src/yb/yql/pggate/ybc_pggate.h

@@ -692,6 +692,8 @@ YBCStatus YBCGetIndexBackfillProgress(YBCPgOid* index_oids, YBCPgOid* database_o
                                       uint64_t** backfill_statuses,
                                       int num_indexes);
 
+void YBCStartSysTablePrefetchingNoCache();
+
 void YBCStartSysTablePrefetching(
     YBCPgLastKnownCatalogVersionInfo catalog_version,
     YBCPgSysTablePrefetcherCacheMode cache_mode);