feat: use Workers for manifest fetching & optimizations

src/components/Card/Card.tsx

@@ -10,7 +10,6 @@ import {
   parseCSS,
   injectUserCSS,
   generateKey,
-  getAvailableTLD,
 } from "../../logic/Utils";
 import TrashIcon from "../Icons/TrashIcon";
 import DownloadIcon from "../Icons/DownloadIcon";
@@ -379,7 +378,7 @@ class Card extends React.Component<CardProps, {
    */
   async fetchAndInjectUserCSS(theme) {
     try {
-      const tld = await getAvailableTLD();
+      const tld = window.sessionStorage.getItem("marketplace-request-tld") || undefined;
       const userCSS = theme
         ? await parseCSS(this.props.item as CardItem, tld)
         : undefined;

src/components/Grid.tsx

@@ -417,6 +417,7 @@ class Grid extends React.Component<
     // Checks for new Marketplace updates
     fetch(LATEST_RELEASE).then(res => res.json()).then(
       result => {
+        if (result.message) throw result;
         this.setState({
           version: result[0].name,
         });

src/extensions/extension.tsx

@@ -16,8 +16,6 @@ import {
   parseCSS,
   // TODO: there's a slightly different copy of this function in Card.ts?
   injectUserCSS,
-  addToSessionStorage,
-  sleep,
   addExtensionToSpicetifyConfig,
   initAlbumArtBasedColor,
   getAvailableTLD,
@@ -26,15 +24,14 @@ import {
   getBlacklist,
   fetchThemeManifest,
   fetchExtensionManifest,
+  fetchAppManifest,
 } from "../logic/FetchRemotes";
 
-(async () => {
+(async function init() {
   while (!(Spicetify?.LocalStorage && Spicetify?.showNotification)) {
-    await new Promise(resolve => setTimeout(resolve, 100));
+    await new Promise(resolve => setTimeout(resolve, 10));
   }
 
-  const tld = await getAvailableTLD();
-
   // https://github.com/satya164/react-simple-code-editor/issues/86
   const reactSimpleCodeEditorFix = document.createElement("script");
   reactSimpleCodeEditorFix.innerHTML = "const global = globalThis;";
@@ -52,6 +49,8 @@ import {
     version: MARKETPLACE_VERSION,
   };
 
+  const tld = await getAvailableTLD();
+
   const initializeExtension = (extensionKey: string) => {
     const extensionManifest = getLocalStorageDataFromKey(extensionKey);
     // Abort if no manifest found or no extension URL (i.e. a theme)
@@ -146,16 +145,30 @@ import {
 
   console.log("Loaded Marketplace extension");
 
+  const installedSnippetKeys = getLocalStorageDataFromKey(LOCALSTORAGE_KEYS.installedSnippets, []);
+  const installedSnippets = installedSnippetKeys.map((key) => getLocalStorageDataFromKey(key));
+  initializeSnippets(installedSnippets);
+
+  if (!tld) {
+    if (window.navigator.onLine) {
+      console.error(new Error("Unable to connect to the CDN, please check your Internet configuration."));
+      Spicetify.showNotification("Marketplace is unable to connect to the CDN. Please check your Internet configuration.", true, 5000);
+    } else {
+      // Reload Marketplace extension in case the user couldn't connect to the CDN because they were offline
+      window.addEventListener("online", init, { once: true });
+    }
+
+    return;
+  }
+
+  window.sessionStorage.setItem("marketplace-request-tld", tld);
+
   // Save to Spicetify.Config for use when removing a theme
   Spicetify.Config.local_theme = Spicetify.Config.current_theme;
   Spicetify.Config.local_color_scheme = Spicetify.Config.color_scheme;
   const installedThemeKey = localStorage.getItem(LOCALSTORAGE_KEYS.themeInstalled);
   if (installedThemeKey) initializeTheme(installedThemeKey);
 
-  const installedSnippetKeys = getLocalStorageDataFromKey(LOCALSTORAGE_KEYS.installedSnippets, []);
-  const installedSnippets = installedSnippetKeys.map((key) => getLocalStorageDataFromKey(key));
-  initializeSnippets(installedSnippets);
-
   const installedExtensions = getLocalStorageDataFromKey(LOCALSTORAGE_KEYS.installedExtensions, []);
   installedExtensions.forEach((extensionKey) => initializeExtension(extensionKey));
 })();
@@ -169,16 +182,20 @@ import {
 async function queryRepos(type: RepoType, pageNum = 1) {
   const BLACKLIST = window.sessionStorage.getItem("marketplace:blacklist");
 
-  let url = `https://api.github.com/search/repositories?per_page=${ITEMS_PER_REQUEST}`;
-  if (type === "extension") url += `&q=${encodeURIComponent("topic:spicetify-extensions")}`;
-  else if (type === "theme") url += `&q=${encodeURIComponent("topic:spicetify-themes")}`;
+  let url = `https://api.github.com/search/repositories?per_page=${ITEMS_PER_REQUEST}&q=${encodeURIComponent(`topic:spicetify-${type}s`)}`;
   if (pageNum) url += `&page=${pageNum}`;
 
-  const allRepos = await fetch(url).then(res => res.json()).catch(() => []);
-  if (!allRepos.items) {
-    Spicetify.showNotification("Too Many Requests, Cool Down.", true);
+  const allRepos = JSON.parse(window.sessionStorage.getItem(`spicetify-${type}s-page-${pageNum}`) || "null") || await fetch(url)
+    .then(res => res.json())
+    .catch(() => null);
+
+  if (!allRepos?.items) {
+    Spicetify.showNotification?.("Too Many Requests, Cool Down.", true);
+    return { items: [] };
   }
 
+  window.sessionStorage.setItem(`spicetify-${type}s-page-${pageNum}`, JSON.stringify(allRepos));
+
   const filteredResults = {
     ...allRepos,
     page_count: allRepos.items.length,
@@ -199,7 +216,7 @@ async function loadPageRecursive(type: RepoType, pageNum: number) {
   appendInformationToLocalStorage(pageOfRepos, type);
 
   // Sets the amount of items that have thus been fetched
-  const soFarResults = ITEMS_PER_REQUEST * (pageNum - 1) + pageOfRepos.page_count;
+  const soFarResults = ITEMS_PER_REQUEST * pageNum + pageOfRepos.page_count;
   console.debug({ pageOfRepos });
   const remainingResults = pageOfRepos.total_count - soFarResults;
 
@@ -221,8 +238,9 @@ async function loadPageRecursive(type: RepoType, pageNum: number) {
   // Begin by getting the themes and extensions from github
   // const [extensionReposArray, themeReposArray] = await Promise.all([
   await Promise.all([
-    loadPageRecursive("extension", 1),
-    loadPageRecursive("theme", 1),
+    loadPageRecursive("extension", 0),
+    loadPageRecursive("theme", 0),
+    loadPageRecursive("app", 0),
   ]);
 
   // let extensionsNextPage = 1;
@@ -241,13 +259,8 @@ async function loadPageRecursive(type: RepoType, pageNum: number) {
 async function appendInformationToLocalStorage(array, type: RepoType) {
   // This system should make it so themes and extensions are stored concurrently
   for (const repo of array.items) {
-    // console.log(repo);
-    const data = (type === "theme")
-      ? await fetchThemeManifest(repo.contents_url, repo.default_branch, repo.stargazers_count)
-      : await fetchExtensionManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
-    if (data) {
-      addToSessionStorage(data);
-      await sleep(5000);
-    }
+    if (type === "theme") await fetchThemeManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
+    else if (type === "extension") await fetchExtensionManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
+    else if (type === "app") await fetchAppManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
   }
 }

src/logic/FetchRemotes.ts

@@ -25,11 +25,17 @@ export async function getTaggedRepos(tag: RepoTopic, page = 1, BLACKLIST:string[
   // Sorting params (not implemented for Marketplace yet)
   // if (sortConfig.by.match(/top|controversial/) && sortConfig.time) {
   //     url += `&t=${sortConfig.time}`
-  const allRepos = await fetch(url).then(res => res.json()).catch(() => []);
-  if (!allRepos.items) {
+  const allRepos = JSON.parse(window.sessionStorage.getItem(`${tag}-page-${page}`) || "null") || await fetch(url)
+    .then(res => res.json())
+    .catch(() => null);
+
+  if (!allRepos?.items) {
     Spicetify.showNotification("Too Many Requests, Cool Down.", true);
-    return;
+    return { items: [] };
   }
+
+  window.sessionStorage.setItem(`${tag}-page-${page}`, JSON.stringify(allRepos));
+
   const filteredResults = {
     ...allRepos,
     // Include count of all items on the page, since we're filtering the blacklist below,
@@ -41,6 +47,32 @@ export async function getTaggedRepos(tag: RepoTopic, page = 1, BLACKLIST:string[
   return filteredResults;
 }
 
+// Workaround for not spamming console with 404s
+const script = `
+  self.addEventListener('message', async (event) => {
+    const url = event.data;
+    const response = await fetch(url);
+    const data = await response.json().catch(() => null);
+    self.postMessage(data);
+  });
+`;
+const blob = new Blob([script], { type: "application/javascript" });
+const workerURL = URL.createObjectURL(blob);
+
+async function fetchRepoManifest(url: string) {
+  const worker = new Worker(workerURL);
+  return new Promise((resolver) => {
+    const resolve = (data) => {
+      worker.terminate();
+      resolver(data);
+    };
+
+    worker.postMessage(url);
+    worker.addEventListener("message", (event) => resolve(event.data), { once: true });
+    worker.addEventListener("error", () => resolve(null), { once: true });
+  });
+}
+
 // TODO: add try/catch here?
 // TODO: can we add a return type here?
 /**
@@ -51,17 +83,20 @@ export async function getTaggedRepos(tag: RepoTopic, page = 1, BLACKLIST:string[
 * @returns The manifest object
 */
 async function getRepoManifest(user: string, repo: string, branch: string) {
-  const sessionStorageItem = window.sessionStorage.getItem(`${user}-${repo}`);
-  const failedSessionStorageItems = window.sessionStorage.getItem("noManifests");
+  const key = `${user}-${repo}`;
+  const sessionStorageItem = window.sessionStorage.getItem(key);
+  const failedSessionStorageItems = JSON.parse(window.sessionStorage.getItem("noManifests") || "[]");
   if (sessionStorageItem) return JSON.parse(sessionStorageItem);
 
   const url = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/manifest.json`;
-  if (failedSessionStorageItems?.includes(url)) return null;
+  if (failedSessionStorageItems.includes(url)) return null;
+
+  let manifest = await fetchRepoManifest(url);
+
+  if (!manifest) return addToSessionStorage([url], "noManifests");
+  if (!Array.isArray(manifest)) manifest = [manifest];
 
-  const manifest = await fetch(url).then(res => res.json()).catch(
-    () => addToSessionStorage([url], "noManifests"),
-  );
-  if (manifest) window.sessionStorage.setItem(`${user}-${repo}`, JSON.stringify(manifest));
+  addToSessionStorage(manifest, key);
 
   return manifest;
 }
@@ -77,16 +112,12 @@ async function getRepoManifest(user: string, repo: string, branch: string) {
 export async function fetchExtensionManifest(contents_url: string, branch: string, stars: number, hideInstalled = false) {
   try {
     // TODO: use the original search full_name ("theRealPadster/spicetify-hide-podcasts") or something to get the url better?
-    let manifests;
     const regex_result = contents_url.match(/https:\/\/api\.github\.com\/repos\/(?<user>.+)\/(?<repo>.+)\/contents/);
     // TODO: err handling?
     if (!regex_result || !regex_result.groups) return null;
     const { user, repo } = regex_result.groups;
 
-    manifests = await getRepoManifest(user, repo, branch);
-
-    // If the manifest returned is not an array, initialize it as one
-    if (!Array.isArray(manifests)) manifests = [manifests];
+    const manifests = await getRepoManifest(user, repo, branch);
 
     // Manifest is initially parsed
     const parsedManifests: CardItem[] = manifests.reduce((accum, manifest) => {
@@ -129,9 +160,7 @@ export async function fetchExtensionManifest(contents_url: string, branch: strin
     }, []);
 
     return parsedManifests;
-  }
-  catch (err) {
-    // console.warn(contents_url, err);
+  } catch {
     return null;
   }
 }
@@ -146,16 +175,12 @@ export async function fetchExtensionManifest(contents_url: string, branch: strin
 */
 export async function fetchThemeManifest(contents_url: string, branch: string, stars: number) {
   try {
-    let manifests;
     const regex_result = contents_url.match(/https:\/\/api\.github\.com\/repos\/(?<user>.+)\/(?<repo>.+)\/contents/);
     // TODO: err handling?
     if (!regex_result || !regex_result.groups) return null;
     const { user, repo } = regex_result.groups;
 
-    manifests = await getRepoManifest(user, repo, branch);
-
-    // If the manifest returned is not an array, initialize it as one
-    if (!Array.isArray(manifests)) manifests = [manifests];
+    const manifests = await getRepoManifest(user, repo, branch);
 
     // Manifest is initially parsed
     // const parsedManifests: ThemeCardItem[] = manifests.reduce((accum, manifest) => {
@@ -196,9 +221,7 @@ export async function fetchThemeManifest(contents_url: string, branch: string, s
       return accum;
     }, []);
     return parsedManifests;
-  }
-  catch (err) {
-    // console.warn(contents_url, err);
+  } catch {
     return null;
   }
 }
@@ -213,16 +236,12 @@ export async function fetchThemeManifest(contents_url: string, branch: string, s
 export async function fetchAppManifest(contents_url: string, branch: string, stars: number) {
   try {
     // TODO: use the original search full_name ("theRealPadster/spicetify-hide-podcasts") or something to get the url better?
-    let manifests;
     const regex_result = contents_url.match(/https:\/\/api\.github\.com\/repos\/(?<user>.+)\/(?<repo>.+)\/contents/);
     // TODO: err handling?
     if (!regex_result || !regex_result.groups) return null;
     const { user, repo } = regex_result.groups;
 
-    manifests = await getRepoManifest(user, repo, branch);
-
-    // If the manifest returned is not an array, initialize it as one
-    if (!Array.isArray(manifests)) manifests = [manifests];
+    const manifests = await getRepoManifest(user, repo, branch);
 
     // Manifest is initially parsed
     const parsedManifests: CardItem[] = manifests.reduce((accum, manifest) => {
@@ -263,9 +282,7 @@ export async function fetchAppManifest(contents_url: string, branch: string, sta
     }, []);
 
     return parsedManifests;
-  }
-  catch (err) {
-    // console.warn(contents_url, err);
+  } catch {
     return null;
   }
 }

src/logic/Utils.ts

@@ -417,9 +417,11 @@ export const initAlbumArtBasedColor = (scheme: ColourScheme) => {
   });
 };
 
-export const parseCSS = async (themeData: CardItem, tld = "net") => {
+export const parseCSS = async (themeData: CardItem, tld?: string) => {
   if (!themeData.cssURL) throw new Error("No CSS URL provided");
 
+  tld ||= await getAvailableTLD();
+
   const userCssUrl = isGithubRawUrl(themeData.cssURL)
   // TODO: this should probably be the URL stored in localstorage actually (i.e. put this url in localstorage)
     ? `https://cdn.jsdelivr.${tld}/gh/${themeData.user}/${themeData.repo}@${themeData.branch}/${themeData.manifest.usercss}`
@@ -479,12 +481,13 @@ export const getParamsFromGithubRaw = (url: string) => {
 export function addToSessionStorage(items, key?) {
   if (!items) return;
   items.forEach((item) => {
-    if (!key) key = `${items.user}-${items.repo}`;
+    const itemKey = key || `${item.user}-${item.repo}`;
+
     // If the key already exists, it will append to it instead of overwriting it
-    const existing = window.sessionStorage.getItem(key);
+    const existing = window.sessionStorage.getItem(itemKey);
     const parsed = existing ? JSON.parse(existing) : [];
     parsed.push(item);
-    window.sessionStorage.setItem(key, JSON.stringify(parsed));
+    window.sessionStorage.setItem(itemKey, JSON.stringify(parsed));
   });
 }
 export function getInvalidCSS(): string[] {
@@ -581,11 +584,15 @@ export const addExtensionToSpicetifyConfig = (main?: string) => {
 
 // Make a ping to the jsdelivr CDN to check if the user has an internet connection
 export async function getAvailableTLD() {
-  try {
-    const response = await fetch("https://cdn.jsdelivr.net", { redirect: "manual" });
-    if (response.type === "opaqueredirect") return "net";
-    return "xyz";
-  } catch (err) {
-    return "xyz";
+  const tlds = ["net", "xyz"];
+
+  for (const tld of tlds) {
+    try {
+      const response = await fetch(`https://cdn.jsdelivr.${tld}`, { redirect: "manual", cache: "no-cache" });
+      if (response.type === "opaqueredirect") return tld;
+    } catch (err) {
+      console.error(err);
+      continue;
+    }
   }
 }

src/styles/components/_grid.scss

@@ -31,12 +31,8 @@
 }
 
 .marketplace-header__left {
-  position: fixed;
-  left: 16px;
-
-  @media (min-width: 1024px) {
-    left: 32px;
-  }
+  position: absolute;
+  left: 0;
 }
 
 .marketplace-grid {