feat: use Workers for manifest fetching & optimizations

src/components/Grid.tsx

@@ -340,15 +340,25 @@ class Grid extends React.Component<
     this.setState({ rest: false });
     quantity += this.cardList.length;
 
-    this.requestPage = await this.loadPage(queue);
+    try {
+      this.requestPage = await this.loadPage(queue);
+    } catch (e) {
+      console.error(e);
+      this.requestPage = -1;
+    }
 
     while (
       this.requestPage &&
       this.requestPage !== -1 &&
       this.cardList.length < quantity &&
       !this.state.endOfList
     ) {
-      this.requestPage = await this.loadPage(queue);
+      try {
+        this.requestPage = await this.loadPage(queue);
+      } catch (e) {
+        console.error(e);
+        this.requestPage = -1;
+      }
     }
 
     if (this.requestPage === -1) {

src/extensions/extension.tsx

@@ -239,15 +239,22 @@
 })();
 
 async function appendInformationToLocalStorage(array, type: RepoType) {
-  // This system should make it so themes and extensions are stored concurrently
-  for (const repo of array.items) {
-    // console.log(repo);
-    const data = (type === "theme")
-      ? await fetchThemeManifest(repo.contents_url, repo.default_branch, repo.stargazers_count)
-      : await fetchExtensionManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
-    if (data) {
-      addToSessionStorage(data);
-      await sleep(5000);
+  try {
+    // This system should make it so themes and extensions are stored concurrently
+    for (const repo of array.items) {
+      // console.log(repo);
+      const data = (type === "theme")
+        ? await fetchThemeManifest(repo.contents_url, repo.default_branch, repo.stargazers_count)
+        : await fetchExtensionManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
+      if (data) {
+        addToSessionStorage(data);
+        await sleep(5000);
+      }
     }
+  } catch (err: any) {
+    if (err.message.includes("API rate limit exceeded")) return Spicetify.showNotification("Too Many Requests, Cool Down.", true);
+
+    Spicetify.showNotification("Error loading themes and extensions", true);
+    console.error(err);
   }
 }

src/logic/FetchRemotes.ts

@@ -1,5 +1,5 @@
 import { BLACKLIST_URL, ITEMS_PER_REQUEST } from "../constants";
-import { CardItem, Snippet } from "../types/marketplace-types";
+import { CardItem, GithubContents, GithubMessage, Snippet } from "../types/marketplace-types";
 import { addToSessionStorage, processAuthors } from "./Utils";
 
 import { RepoTopic } from "../types/marketplace-types";
@@ -28,7 +28,7 @@
   const allRepos = await fetch(url).then(res => res.json()).catch(() => []);
   if (!allRepos.items) {
     Spicetify.showNotification("Too Many Requests, Cool Down.", true);
-    return;
+    return { items: [] };
   }
   const filteredResults = {
     ...allRepos,
@@ -41,6 +41,20 @@
   return filteredResults;
 }
 
+async function fetchRepoManifest(contents: GithubContents[] | GithubMessage) {
+  if (Array.isArray(contents)) {
+    const manifest = contents.find(item => item.name === "manifest.json");
+    if (!manifest) throw new Error("No manifest");
+
+    const manifestContents = await fetch(manifest.download_url).then(res => res.json()).catch(() => null);
+    if (!manifestContents) return null;
+
+    return manifestContents;
+  } else {
+    return contents.message;
+  }
+}
+
 // TODO: add try/catch here?
 // TODO: can we add a return type here?
 /**
@@ -50,17 +64,18 @@
 * @param branch Default branch name (e.g. main or master)
 * @returns The manifest object
 */
-async function getRepoManifest(user: string, repo: string, branch: string) {
+async function getRepoManifest(user: string, repo: string, contentsUrl: string) {
   const sessionStorageItem = window.sessionStorage.getItem(`${user}-${repo}`);
   const failedSessionStorageItems = window.sessionStorage.getItem("noManifests");
   if (sessionStorageItem) return JSON.parse(sessionStorageItem);
+  if (failedSessionStorageItems?.includes(contentsUrl)) return null;
 
-  const url = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/manifest.json`;
-  if (failedSessionStorageItems?.includes(url)) return null;
+  const manifest = await fetch(contentsUrl)
+    .then(res => res.json())
+    .then(fetchRepoManifest)
+    .catch(() => addToSessionStorage([contentsUrl], "noManifests"));
 
-  const manifest = await fetch(url).then(res => res.json()).catch(
-    () => addToSessionStorage([url], "noManifests"),
-  );
+  if (typeof manifest === "string") throw new Error(manifest);
   if (manifest) window.sessionStorage.setItem(`${user}-${repo}`, JSON.stringify(manifest));
 
   return manifest;
@@ -83,7 +98,7 @@
     if (!regex_result || !regex_result.groups) return null;
     const { user, repo } = regex_result.groups;
 
-    manifests = await getRepoManifest(user, repo, branch);
+    manifests = await getRepoManifest(user, repo, regex_result[0]);
 
     // If the manifest returned is not an array, initialize it as one
     if (!Array.isArray(manifests)) manifests = [manifests];
@@ -130,8 +145,8 @@
 
     return parsedManifests;
   }
-  catch (err) {
-    // console.warn(contents_url, err);
+  catch (err: any) {
+    if (err.message.includes("API rate limit exceeded")) throw err;
     return null;
   }
 }
@@ -152,7 +167,7 @@
     if (!regex_result || !regex_result.groups) return null;
     const { user, repo } = regex_result.groups;
 
-    manifests = await getRepoManifest(user, repo, branch);
+    manifests = await getRepoManifest(user, repo, regex_result[0]);
 
     // If the manifest returned is not an array, initialize it as one
     if (!Array.isArray(manifests)) manifests = [manifests];
@@ -197,8 +212,8 @@
     }, []);
     return parsedManifests;
   }
-  catch (err) {
-    // console.warn(contents_url, err);
+  catch (err: any) {
+    if (err.message.includes("API rate limit exceeded")) throw err;
     return null;
   }
 }
@@ -219,7 +234,7 @@
     if (!regex_result || !regex_result.groups) return null;
     const { user, repo } = regex_result.groups;
 
-    manifests = await getRepoManifest(user, repo, branch);
+    manifests = await getRepoManifest(user, repo, regex_result[0]);
 
     // If the manifest returned is not an array, initialize it as one
     if (!Array.isArray(manifests)) manifests = [manifests];
@@ -264,8 +279,8 @@
 
     return parsedManifests;
   }
-  catch (err) {
-    // console.warn(contents_url, err);
+  catch (err: any) {
+    if (err.message.includes("API rate limit exceeded")) throw err;
     return null;
   }
 }

src/types/marketplace-types.d.ts

@@ -162,3 +162,25 @@ export type Config = {
     activeScheme?: string | null;
   },
 };
+
+export type GithubContents = {
+  name: string;
+  path: string;
+  sha: string;
+  size: number;
+  url: string;
+  html_url: string;
+  git_url: string;
+  download_url: string;
+  type: "file" | "dir" | "symlink";
+  _links: {
+    self: string;
+    git: string;
+    html: string;
+  };
+};
+
+export type GithubMessage = {
+  message: string;
+  documentation_url: string;
+};