fix(insights): use RPC as source of truth for publisher list

apps/insights/src/app/api/pyth/get-publishers/[symbol]/route.ts

@@ -8,7 +8,7 @@ import {
   ClusterToName,
   toCluster,
 } from "../../../../../services/pyth";
-import { getPublishersForCluster } from "../../../../../services/pyth/get-publishers-for-cluster";
+import { getPublishersByFeedForCluster } from "../../../../../services/pyth/get-publishers-for-cluster";
 
 export const GET = async (
   request: NextRequest,
@@ -32,7 +32,7 @@ export const GET = async (
     });
   }
 
-  const map = await getPublishersForCluster(cluster);
+  const map = await getPublishersByFeedForCluster(cluster);
 
   return NextResponse.json(map[symbol] ?? []);
 };

apps/insights/src/app/api/pyth/get-publishers/route.ts

@@ -0,0 +1,25 @@
+import { NextRequest, NextResponse } from "next/server";
+import { z } from "zod";
+
+import type { Cluster } from "../../../../services/pyth";
+import { CLUSTER_NAMES, toCluster } from "../../../../services/pyth";
+import { getFeedsByPublisherForCluster } from "../../../../services/pyth/get-publishers-for-cluster";
+
+export const GET = async (request: NextRequest) => {
+  const cluster = clusterSchema.safeParse(
+    request.nextUrl.searchParams.get("cluster"),
+  );
+
+  return cluster.success
+    ? NextResponse.json(await getPublishers(cluster.data))
+    : new Response("Invalid params", { status: 400 });
+};
+
+const clusterSchema = z
+  .enum(CLUSTER_NAMES)
+  .transform((value) => toCluster(value));
+
+const getPublishers = async (cluster: Cluster) =>
+  Object.entries(await getFeedsByPublisherForCluster(cluster)).map(
+    ([key, feeds]) => ({ key, permissionedFeeds: feeds.length }),
+  );

apps/insights/src/components/Publisher/get-price-feeds.tsx

@@ -1,12 +1,12 @@
 import { getFeedsForPublisherRequest } from "../../server/pyth";
-import { getRankingsByPublisher } from "../../services/clickhouse";
+import { getFeedRankingsByPublisher } from "../../services/clickhouse";
 import { Cluster, ClusterToName } from "../../services/pyth";
 import { getStatus } from "../../status";
 
 export const getPriceFeeds = async (cluster: Cluster, key: string) => {
   const [feeds, rankings] = await Promise.all([
     getFeedsForPublisherRequest(cluster, key),
-    getRankingsByPublisher(key),
+    getFeedRankingsByPublisher(key),
   ]);
   return feeds.map((feed) => {
     const ranking = rankings.find(

apps/insights/src/components/Publisher/layout.tsx

@@ -15,8 +15,8 @@ import { notFound } from "next/navigation";
 import type { ReactNode } from "react";
 import { Suspense } from "react";
 
+import { getPublishersWithRankings } from "../../get-publishers-with-rankings";
 import {
-  getPublishers,
   getPublisherRankingHistory,
   getPublisherAverageScoreHistory,
 } from "../../services/clickhouse";
@@ -38,13 +38,13 @@ import {
   ExplainActive,
   ExplainInactive,
 } from "../Explanations";
+import { FormattedDate } from "../FormattedDate";
 import { FormattedNumber } from "../FormattedNumber";
 import { PublisherIcon } from "../PublisherIcon";
 import { PublisherKey } from "../PublisherKey";
 import { PublisherTag } from "../PublisherTag";
 import { getPriceFeeds } from "./get-price-feeds";
 import styles from "./layout.module.scss";
-import { FormattedDate } from "../FormattedDate";
 import { FormattedTokens } from "../FormattedTokens";
 import { SemicircleMeter } from "../SemicircleMeter";
 import { TabPanel, TabRoot, Tabs } from "../Tabs";
@@ -365,7 +365,7 @@ const ActiveFeedsCard = async ({
   publisherKey: string;
 }) => {
   const [publishers, priceFeeds] = await Promise.all([
-    getPublishers(cluster),
+    getPublishersWithRankings(cluster),
     getPriceFeeds(cluster, publisherKey),
   ]);
   const publisher = publishers.find(
@@ -391,8 +391,8 @@ type ActiveFeedsCardImplProps =
       isLoading?: false | undefined;
       cluster: Cluster;
       publisherKey: string;
-      activeFeeds: number;
-      inactiveFeeds: number;
+      activeFeeds?: number | undefined;
+      inactiveFeeds?: number | undefined;
       allFeeds: number;
     };
 
@@ -435,33 +435,27 @@ const ActiveFeedsCardImpl = (props: ActiveFeedsCardImplProps) => (
       )
     }
     miniStat1={
-      props.isLoading ? (
-        <Skeleton width={10} />
-      ) : (
-        <>
-          <FormattedNumber
-            maximumFractionDigits={1}
-            value={(100 * props.activeFeeds) / props.allFeeds}
-          />
-          %
-        </>
-      )
+      <RankingMiniStat
+        {...(props.isLoading
+          ? { isLoading: true }
+          : {
+              stat: props.activeFeeds,
+              allFeeds: props.allFeeds,
+            })}
+      />
     }
     miniStat2={
-      props.isLoading ? (
-        <Skeleton width={10} />
-      ) : (
-        <>
-          <FormattedNumber
-            maximumFractionDigits={1}
-            value={(100 * props.inactiveFeeds) / props.allFeeds}
-          />
-          %
-        </>
-      )
+      <RankingMiniStat
+        {...(props.isLoading
+          ? { isLoading: true }
+          : {
+              stat: props.inactiveFeeds,
+              allFeeds: props.allFeeds,
+            })}
+      />
     }
   >
-    {!props.isLoading && (
+    {!props.isLoading && props.activeFeeds !== undefined && (
       <Meter
         value={props.activeFeeds}
         maxValue={props.allFeeds}
@@ -471,6 +465,33 @@ const ActiveFeedsCardImpl = (props: ActiveFeedsCardImplProps) => (
   </StatCard>
 );
 
+const RankingMiniStat = (
+  props:
+    | { isLoading: true }
+    | {
+        isLoading?: false | undefined;
+        stat: number | undefined;
+        allFeeds: number;
+      },
+) => {
+  if (props.isLoading) {
+    return <Skeleton width={10} />;
+  } else if (props.stat === undefined) {
+    // eslint-disable-next-line unicorn/no-null
+    return null;
+  } else {
+    return (
+      <>
+        <FormattedNumber
+          maximumFractionDigits={1}
+          value={(100 * props.stat) / props.allFeeds}
+        />
+        %
+      </>
+    );
+  }
+};
+
 const OisPoolCard = async ({ publisherKey }: { publisherKey: string }) => {
   const [publisherPoolData, publisherCaps] = await Promise.all([
     getPublisherPoolData(),

apps/insights/src/components/Publisher/performance.tsx

@@ -16,7 +16,7 @@ import type { ReactNode, ComponentProps } from "react";
 import { getPriceFeeds } from "./get-price-feeds";
 import styles from "./performance.module.scss";
 import { TopFeedsTable } from "./top-feeds-table";
-import { getPublishers } from "../../services/clickhouse";
+import { getPublishersWithRankings } from "../../get-publishers-with-rankings";
 import type { Cluster } from "../../services/pyth";
 import { ClusterToName, parseCluster } from "../../services/pyth";
 import { Status } from "../../status";
@@ -48,7 +48,7 @@ export const Performance = async ({ params }: Props) => {
     notFound();
   }
   const [publishers, priceFeeds] = await Promise.all([
-    getPublishers(parsedCluster),
+    getPublishersWithRankings(parsedCluster),
     getPriceFeeds(parsedCluster, key),
   ]);
   const slicedPublishers = sliceAround(
@@ -63,7 +63,7 @@ export const Performance = async ({ params }: Props) => {
       prefetch: false,
       nameAsString: knownPublisher?.name ?? publisher.key,
       data: {
-        ranking: (
+        ranking: (publisher.rank !== undefined || publisher.key === key) && (
           <Ranking isCurrent={publisher.key === key} className={styles.ranking}>
             {publisher.rank}
           </Ranking>
@@ -86,7 +86,7 @@ export const Performance = async ({ params }: Props) => {
             {publisher.inactiveFeeds}
           </Link>
         ),
-        averageScore: (
+        averageScore: publisher.averageScore !== undefined && (
           <Score width={PUBLISHER_SCORE_WIDTH} score={publisher.averageScore} />
         ),
         name: (

apps/insights/src/components/Publishers/index.tsx

@@ -7,7 +7,7 @@ import { lookup as lookupPublisher } from "@pythnetwork/known-publishers";
 
 import styles from "./index.module.scss";
 import { PublishersCard } from "./publishers-card";
-import { getPublishers } from "../../services/clickhouse";
+import { getPublishersWithRankings } from "../../get-publishers-with-rankings";
 import { getPublisherCaps } from "../../services/hermes";
 import { Cluster } from "../../services/pyth";
 import {
@@ -27,12 +27,15 @@ const INITIAL_REWARD_POOL_SIZE = 60_000_000_000_000n;
 export const Publishers = async () => {
   const [pythnetPublishers, pythtestConformancePublishers, oisStats] =
     await Promise.all([
-      getPublishers(Cluster.Pythnet),
-      getPublishers(Cluster.PythtestConformance),
+      getPublishersWithRankings(Cluster.Pythnet),
+      getPublishersWithRankings(Cluster.PythtestConformance),
       getOisStats(),
     ]);
-  const rankingTime = pythnetPublishers[0]?.timestamp;
-  const scoreTime = pythnetPublishers[0]?.scoreTime;
+  const rankedPublishers = pythnetPublishers.filter(
+    (publisher) => publisher.scoreTime !== undefined,
+  );
+  const rankingTime = rankedPublishers[0]?.timestamp;
+  const scoreTime = rankedPublishers[0]?.scoreTime;
 
   return (
     <div className={styles.publishers}>
@@ -65,10 +68,10 @@ export const Publishers = async () => {
           corner={<ExplainAverage scoreTime={scoreTime} />}
           className={styles.statCard ?? ""}
           stat={(
-            pythnetPublishers.reduce(
-              (sum, publisher) => sum + publisher.averageScore,
+            rankedPublishers.reduce(
+              (sum, publisher) => sum + (publisher.averageScore ?? 0),
               0,
-            ) / pythnetPublishers.length
+            ) / rankedPublishers.length
           ).toFixed(2)}
         />
         <PublishersCard
@@ -149,7 +152,7 @@ const toTableRow = ({
   permissionedFeeds,
   activeFeeds,
   averageScore,
-}: Awaited<ReturnType<typeof getPublishers>>[number]) => {
+}: Awaited<ReturnType<typeof getPublishersWithRankings>>[number]) => {
   const knownPublisher = lookupPublisher(key);
   return {
     id: key,

apps/insights/src/components/Publishers/publishers-card.tsx

@@ -45,10 +45,10 @@ type Props = {
 
 type Publisher = {
   id: string;
-  ranking: number;
+  ranking?: number | undefined;
   permissionedFeeds: number;
-  activeFeeds: number;
-  averageScore: number;
+  activeFeeds?: number | undefined;
+  averageScore?: number | undefined;
 } & (
   | { name: string; icon: ReactNode }
   | { name?: undefined; icon?: undefined }
@@ -100,27 +100,38 @@ const ResolvedPublishersCard = ({
       filter.contains(publisher.id, search) ||
       (publisher.name !== undefined && filter.contains(publisher.name, search)),
     (a, b, { column, direction }) => {
+      const desc = direction === "descending" ? -1 : 1;
+
+      const sortByName =
+        desc * collator.compare(a.name ?? a.id, b.name ?? b.id);
+
+      const sortByRankingField = (
+        column: "ranking" | "activeFeeds" | "averageScore",
+      ) => {
+        if (a[column] === undefined) {
+          return b[column] === undefined ? sortByName : 1;
+        } else {
+          return b[column] === undefined ? -1 : desc * (a[column] - b[column]);
+        }
+      };
+
       switch (column) {
+        case "permissionedFeeds": {
+          return desc * (a[column] - b[column]);
+        }
+
         case "ranking":
-        case "permissionedFeeds":
         case "activeFeeds":
         case "averageScore": {
-          return (
-            (direction === "descending" ? -1 : 1) * (a[column] - b[column])
-          );
+          return sortByRankingField(column);
         }
 
         case "name": {
-          return (
-            (direction === "descending" ? -1 : 1) *
-            collator.compare(a.name ?? a.id, b.name ?? b.id)
-          );
+          return sortByName;
         }
 
         default: {
-          return (
-            (direction === "descending" ? -1 : 1) * (a.ranking - b.ranking)
-          );
+          return sortByRankingField("ranking");
         }
       }
     },
@@ -144,7 +155,7 @@ const ResolvedPublishersCard = ({
           textValue: publisher.name ?? id,
           prefetch: false,
           data: {
-            ranking: <Ranking>{ranking}</Ranking>,
+            ranking: ranking !== undefined && <Ranking>{ranking}</Ranking>,
             name: (
               <PublisherTag
                 publisherKey={id}
@@ -164,7 +175,7 @@ const ResolvedPublishersCard = ({
                 {activeFeeds}
               </Link>
             ),
-            averageScore: (
+            averageScore: averageScore !== undefined && (
               <Score score={averageScore} width={PUBLISHER_SCORE_WIDTH} />
             ),
           },

apps/insights/src/components/Root/index.tsx

@@ -3,18 +3,18 @@ import { lookup as lookupPublisher } from "@pythnetwork/known-publishers";
 import { NuqsAdapter } from "nuqs/adapters/next/app";
 import type { ReactNode } from "react";
 
+import { SearchButton as SearchButtonImpl } from "./search-button";
 import {
   AMPLITUDE_API_KEY,
   ENABLE_ACCESSIBILITY_REPORTING,
   GOOGLE_ANALYTICS_ID,
 } from "../../config/server";
+import { getPublishersWithRankings } from "../../get-publishers-with-rankings";
 import { LivePriceDataProvider } from "../../hooks/use-live-price-data";
-import { getPublishers } from "../../services/clickhouse";
 import { Cluster } from "../../services/pyth";
 import { getFeeds } from "../../services/pyth/get-feeds";
 import { PriceFeedIcon } from "../PriceFeedIcon";
 import { PublisherIcon } from "../PublisherIcon";
-import { SearchButton as SearchButtonImpl } from "./search-button";
 
 export const TABS = [
   { segment: "", children: "Overview" },
@@ -53,7 +53,7 @@ const SearchButton = async () => {
 };
 
 const getPublishersForSearchDialog = async (cluster: Cluster) => {
-  const publishers = await getPublishers(cluster);
+  const publishers = await getPublishersWithRankings(cluster);
   return publishers.map((publisher) => {
     const knownPublisher = lookupPublisher(publisher.key);