Migrate sync IO utilities to async (#211)

* Migrate sync IO utilities to async

* Add loading components

* remove unused imports

app/contributors/[slug]/loading.tsx

@@ -0,0 +1,5 @@
+import LoadingText from "@/components/LoadingText";
+
+export default function Loading() {
+  return <LoadingText text="Fetching contributors profile" />;
+}

app/contributors/[slug]/page.tsx

@@ -18,7 +18,8 @@ import { formatDuration } from "@/lib/utils";
 import Markdown from "@/components/Markdown";
 
 export async function generateStaticParams() {
-  return getContributorsSlugs()
+  const slugs = await getContributorsSlugs();
+  return slugs
     .filter((slug) => !slug.file.includes("[bot]"))
     .map((slug) => ({ slug: slug.file.replace(".md", "") }));
 }
@@ -31,7 +32,7 @@ type Params = {
 
 export default async function Contributor({ params }: Params) {
   const { slug } = params;
-  const contributor = getContributorBySlug(slug, true) as Contributor;
+  const contributor = await getContributorBySlug(slug, true);
 
   return (
     <div className="bg-background min-h-screen">

app/feed/loading.tsx

@@ -0,0 +1,5 @@
+import LoadingText from "@/components/LoadingText";
+
+export default function Loading() {
+  return <LoadingText text="Fetching latest events" />;
+}

app/feed/page.tsx

@@ -1,8 +1,10 @@
+"use client";
+
 import GitHubEvent from "@/components/gh_events/GitHubEvent";
 import { IGitHubEvent, combineSimilarPushEvents } from "@/lib/gh_events";
-import FeedPagination from "./FeedPagination";
-
-export const revalidate = 300; // revalidate at most every 5 mins
+import { useEffect, useState } from "react";
+import { scrollTo } from "@/lib/utils";
+import LoadingText from "@/components/LoadingText";
 
 type Props = {
   searchParams: {
@@ -31,31 +33,51 @@ function extractPaginationLinks(text: string) {
   return pageNums;
 }
 
-export default async function FeedPage({ searchParams }: Props) {
-  const page = (searchParams.page && parseInt(searchParams.page)) || 1;
+export default function FeedPage({ searchParams }: Props) {
+  const [lastFetchedPage, setLastFetchedPage] = useState(1);
+  const [events, setEvents] = useState<Record<number, IGitHubEvent[]>>({});
 
-  const res = await fetch(
-    `https://api.github.com/orgs/${process.env.NEXT_PUBLIC_GITHUB_ORG}/events?per_page=1000&page=${page}`,
-  );
+  useEffect(() => {
+    fetch(
+      `https://api.github.com/orgs/${process.env.NEXT_PUBLIC_GITHUB_ORG}/events?per_page=1000&page=${lastFetchedPage}`,
+    )
+      .then((res) => res.json())
+      .then((data: IGitHubEvent[]) => {
+        setEvents((existing) => ({
+          ...existing,
+          page: combineSimilarPushEvents(
+            data.filter(exludeBotEvents).filter(excludeBlacklistedEvents),
+          ),
+        }));
 
-  const linkHeader = res.headers.get("link");
-  const navLinks = linkHeader ? extractPaginationLinks(linkHeader) : undefined;
+        if (lastFetchedPage !== 1) {
+          const lastEvents = events[lastFetchedPage - 1];
+          scrollTo(`gh-event-${lastEvents[lastEvents.length - 1].id}`);
+        }
+      });
+  }, [lastFetchedPage]);
 
-  const data = (await res.json()) as IGitHubEvent[];
-  const events = combineSimilarPushEvents(
-    data.filter(exludeBotEvents).filter(excludeBlacklistedEvents),
-  );
+  const allEvents = ([] as IGitHubEvent[]).concat(...Object.values(events));
+
+  if (!Object.entries(events).length) {
+    return <LoadingText text="Fetching latest events" />;
+  }
 
   return (
     <div className="flow-root max-w-4xl mx-auto my-8 relative">
       <h1 className="text-primary-500 dark:text-white text-4xl">Feed</h1>
       <ul role="list" className="space-y-4 flex flex-col gap-4 mt-10 mb-20">
-        {events.map((e) => (
+        {allEvents.map((e) => (
           <GitHubEvent key={e.id} event={e} />
         ))}
       </ul>
       <div className="flex flex-row justify-center">
-        {navLinks && <FeedPagination navLinks={navLinks} />}
+        <span
+          className="underline cursor-pointer"
+          onClick={() => setLastFetchedPage((p) => p + 1)}
+        >
+          Show more
+        </span>
       </div>
     </div>
   );

app/leaderboard/Leaderboard.tsx

@@ -4,10 +4,7 @@ import LeaderBoardCard from "@/components/contributors/LeaderboardCard";
 import { Contributor } from "@/lib/types";
 import { TbZoomQuestion } from "react-icons/tb";
 import { LeaderboardResultSet } from "./page";
-import TopContributor, {
-  TOP_CONTRIBUTOR_CATEGORIES,
-  TopContributorCategoryKey,
-} from "../../components/contributors/TopContributor";
+import TopContributor from "../../components/contributors/TopContributor";
 import { useState } from "react";
 import { usePathname, useRouter, useSearchParams } from "next/navigation";
 import { getWeekNumber, parseDateRangeSearchParam } from "@/lib/utils";

app/leaderboard/loading.tsx

@@ -0,0 +1,5 @@
+import LoadingText from "@/components/LoadingText";
+
+export default function Loading() {
+  return <LoadingText text="Ranking the contributors" />;
+}

app/leaderboard/page.tsx

@@ -3,11 +3,11 @@ import { Contributor } from "@/lib/types";
 import Leaderboard, { LeaderboardSortKey } from "./Leaderboard";
 import { parseDateRangeSearchParam } from "@/lib/utils";
 
-const getResultSet = (searchParams: PageProps["searchParams"]) => {
+const getResultSet = async (searchParams: PageProps["searchParams"]) => {
   const [start, end] = parseDateRangeSearchParam(searchParams.between);
   const sortBy = searchParams.sortBy ?? "points";
 
-  let data = getContributors().filter((a) => a.highlights.points);
+  let data = (await getContributors()).filter((a) => a.highlights.points);
 
   const filterByRole = (contributor: Contributor) => {
     const roles = searchParams.roles?.split(",") ?? [];
@@ -51,7 +51,7 @@ const getResultSet = (searchParams: PageProps["searchParams"]) => {
   return result;
 };
 
-export type LeaderboardResultSet = ReturnType<typeof getResultSet>;
+export type LeaderboardResultSet = Awaited<ReturnType<typeof getResultSet>>;
 
 type PageProps = {
   searchParams: {
@@ -62,7 +62,8 @@ type PageProps = {
   };
 };
 
-export default function LeaderboardPage({ searchParams }: PageProps) {
-  const resultSet = getResultSet(searchParams);
+export default async function LeaderboardPage({ searchParams }: PageProps) {
+  const resultSet = await getResultSet(searchParams);
+
   return <Leaderboard resultSet={resultSet} />;
 }

app/page.tsx

@@ -8,11 +8,12 @@ import {
   MdOutlineArrowForwardIos,
 } from "react-icons/md";
 
-export default function Home() {
-  const contributors = getContributors().sort(
+export default async function Home() {
+  const contributors = (await getContributors()).sort(
     (a, b) => b.weekSummary.points - a.weekSummary.points,
   );
   const dateRange = getLastWeekDateRangeString();
+
   return (
     <div className="bg-background text-foreground min-h-screen">
       <section className="bg-background border-t dark:border-gray-700 border-gray-300 ">

components/LoadingText.tsx

@@ -0,0 +1,10 @@
+export default function LoadingText({ text }: { text: string }) {
+  return (
+    <div className="flex h-[75vh] items-center justify-center font-mono text-3xl dark:text-primary-200">
+      <span>{text}</span>
+      <span className="animate-[pulse_1s_infinite_100ms]">.</span>
+      <span className="animate-[pulse_1s_infinite_200ms]">.</span>
+      <span className="animate-[pulse_1s_infinite_300ms]">.</span>
+    </div>
+  );
+}

components/filters/Sort.tsx

@@ -2,7 +2,7 @@ import { HiSortAscending, HiSortDescending } from "react-icons/hi";
 
 const Sort = ({
   sortByOptions = [],
-  sortBy = "",
+  sortBy,
   sortDescending = true,
   handleSortOrderChange,
   handleSortByChange,
@@ -37,7 +37,6 @@ const Sort = ({
         >
           {sortByOptions.map((option) => (
             <option
-              selected={option.value === sortBy}
               key={option.value}
               value={option.value}
               className="text-gray-700"

components/gh_events/GitHubEvent.tsx

@@ -170,7 +170,7 @@ export default function GitHubEvent({ event }: { event?: IGitHubEvent }) {
   }
 
   return (
-    <li className="group">
+    <li className="group" id={`gh-event-${event.id}`}>
       <div className="relative pb-4">
         <span
           className="absolute left-5 top-5 -ml-px h-full w-0.5 bg-gray-200 dark:bg-gray-700 group-last:hidden"

lib/api.ts

@@ -1,8 +1,8 @@
-import fs from "fs";
 import { join } from "path";
 import matter from "gray-matter";
 import { Activity, ActivityData, Contributor, Highlights } from "./types";
 import { padZero } from "./utils";
+import { readFile, readdir } from "fs/promises";
 
 const root = join(process.cwd(), "contributors");
 const slackRoot = join(process.cwd(), "data/slack");
@@ -26,31 +26,29 @@ const points = {
 // Opening a PR would give a single point and merging it would give you the other 7 points, making 8 per PR
 // Updating the EOD would get 2 points per day and additional 20 for regular daily updates plus 10 for just missing one
 
-export function formatSlug(slug: string) {
+function formatSlug(slug: string) {
   return slug.replace(/\.md$/, "");
 }
 
-export function formatSlugJSON(slug: string) {
+function formatSlugJSON(slug: string) {
   return slug.replace(/\.json$/, "");
 }
 
-export function getSlackSlugs() {
-  const slackSlugs: Record<string, string> = {};
-  fs.readdirSync(`${slackRoot}`).forEach((file) => {
-    slackSlugs[formatSlugJSON(file)] = file;
-  });
-
-  return slackSlugs;
+async function getSlackSlugs() {
+  const files = await readdir(`${slackRoot}`);
+  return Object.fromEntries(files.map((file) => [formatSlugJSON(file), file]));
 }
 
-let validSlackSlugs = getSlackSlugs();
+let validSlackSlugs: Awaited<ReturnType<typeof getSlackSlugs>> | null = null;
+
+async function getSlackMessages(slackId: string) {
+  validSlackSlugs ??= await getSlackSlugs();
 
-export function getSlackMessages(slackId: string) {
   const filePath = join(slackRoot, `${slackId}.json`);
   let fileContents = [];
   if (validSlackSlugs[slackId]) {
     try {
-      fileContents = JSON.parse(fs.readFileSync(filePath, "utf8"));
+      fileContents = JSON.parse(await readFile(filePath, "utf8"));
     } catch (e) {
       console.log(e);
     }
@@ -72,34 +70,32 @@ export function getSlackMessages(slackId: string) {
   }
 }
 
-export function getContributorsSlugs() {
-  const contributorSlugs: { file: string }[] = [];
-  fs.readdirSync(`${root}`).forEach((file) => {
-    contributorSlugs.push({ file: file });
-  });
-
-  return contributorSlugs;
+export async function getContributorsSlugs() {
+  const files = await readdir(`${root}`);
+  return files.map((file) => ({ file }));
 }
 
-export function getContributorBySlug(file: string, detail = false) {
+export async function getContributorBySlug(file: string, detail = false) {
   const fullPath = join(root, `${formatSlug(file)}.md`);
-  const { data, content } = matter(fs.readFileSync(fullPath, "utf8"));
+  const { data, content } = matter(await readFile(fullPath, "utf8"));
 
   const githubHandle = file.replace(/\.md$/, "");
 
   let activityData = { activity: [] as Activity[] } as ActivityData;
 
   try {
     activityData = JSON.parse(
-      fs.readFileSync(join(githubRoot, `${githubHandle}.json`), "utf8"),
+      await readFile(join(githubRoot, `${githubHandle}.json`), "utf8"),
     );
   } catch (e) {
     console.log(e);
   }
 
+  const slackMessages = await getSlackMessages(data.slack);
+
   activityData = {
     ...activityData,
-    activity: [...activityData.activity, ...getSlackMessages(data.slack)],
+    activity: [...activityData.activity, ...slackMessages],
   };
 
   const weightedActivity = activityData.activity.reduce(
@@ -181,13 +177,14 @@ export function getContributorBySlug(file: string, detail = false) {
   } as Contributor & { summarize: typeof summarize };
 }
 
-export function getContributors(detail = false) {
-  return getContributorsSlugs().map((path) =>
-    getContributorBySlug(path.file, detail),
+export async function getContributors(detail = false) {
+  const slugs = await getContributorsSlugs();
+  return Promise.all(
+    slugs.map((path) => getContributorBySlug(path.file, detail)),
   );
 }
 
-export function getCalendarData(activity: Activity[]) {
+function getCalendarData(activity: Activity[]) {
   const calendarData = activity.reduce(
     (acc, activity) => {
       // Github activity.time ignores milliseconds (*1000)