diff --git a/.gitignore b/.gitignore index ae2bb14..afab2ea 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ tsconfig.tsbuildinfo *.njsproj *.sln *.sw? + +# .env files +.env +.env.* diff --git a/bun.lockb b/bun.lockb index d713d04..c3fdcd0 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index e11e4be..eeb0c7c 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "dataloader": "^2.2.2", "graphql": "^16.8.0", "linkedom": "^0.15.3", - "picocolors": "^1.0.0" + "picocolors": "^1.0.0", + "radix3": "^1.1.2" }, "devDependencies": { "@aklinker1/check": "^1.2.0", diff --git a/src/apis/firefox-api.ts b/src/apis/firefox-api.ts index 0a0b2fd..571339f 100644 --- a/src/apis/firefox-api.ts +++ b/src/apis/firefox-api.ts @@ -1,4 +1,5 @@ import consola from "consola"; +import { buildScreenshotUrl } from "../utils/urls"; export function createFirefoxApiClient() { return { @@ -29,6 +30,13 @@ export function createFirefoxApiClient() { storeUrl: json.url, version: json.current_version.version, dailyActiveUsers: json.average_daily_users, + screenshots: (json.previews as any[]).map( + (preview, i) => ({ + index: i, + rawUrl: preview.image_url, + indexUrl: buildScreenshotUrl("firefox-addons", json.id, i), + }), + ), }; }, }; diff --git a/src/crawlers/__tests__/chrome-crawler.test.ts b/src/crawlers/__tests__/chrome-crawler.test.ts index a840bab..d973344 100644 --- a/src/crawlers/__tests__/chrome-crawler.test.ts +++ b/src/crawlers/__tests__/chrome-crawler.test.ts @@ -22,6 +22,20 @@ describe("Chrome Web Store Crawler", () => { "https://chromewebstore.google.com/detail/github-better-line-counts/ocfdgncpifmegplaglcnglhioflaimkd", version: expect.any(String), weeklyActiveUsers: expect.any(Number), + screenshots: [ + { + index: 0, + indexUrl: + "http://localhost:3000/api/rest/chrome-extensions/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/0", + rawUrl: expect.any(String), + }, + { + index: 1, + indexUrl: + "http://localhost:3000/api/rest/chrome-extensions/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/1", + rawUrl: expect.any(String), + }, + ], }); }); }); diff --git a/src/crawlers/chrome-crawler.ts b/src/crawlers/chrome-crawler.ts index bc6c32b..59e451e 100644 --- a/src/crawlers/chrome-crawler.ts +++ b/src/crawlers/chrome-crawler.ts @@ -1,5 +1,6 @@ import consola from "consola"; import { HTMLAnchorElement, HTMLElement, parseHTML } from "linkedom"; +import { buildScreenshotUrl } from "../utils/urls"; export async function crawlExtension( id: string, @@ -21,7 +22,7 @@ export async function crawlExtension( const { document } = parseHTML(html); // Uncomment to debug HTML - // Bun.write("chrome.html", document.documentElement.outerHTML); + Bun.write("chrome.html", document.documentElement.outerHTML); // Basic metadata const name = metaContent(document, "property=og:title")?.replace( @@ -106,6 +107,23 @@ export async function crawlExtension( // const rating = extractNumber(ratingDiv.title); // "Average rating: 4.78 stars" // const reviewCount = extractNumber(ratingDiv.textContent); // "(1024)" + //
+ const screenshots = [...document.querySelectorAll("div[data-media-url]")] + .filter((div) => div.getAttribute("data-is-video") === "false") + .map((div) => { + const index = Number(div.getAttribute("data-slide-index") || -1); + return { + index, + rawUrl: div.getAttribute("data-media-url") + "=s1280", // "s1280" gets the full resolution + indexUrl: buildScreenshotUrl("chrome-extensions", id, index), + }; + }); + if (name == null) return; if (storeUrl == null) return; if (iconUrl == null) return; @@ -114,6 +132,12 @@ export async function crawlExtension( if (version == null) return; if (shortDescription == null) return; if (longDescription == null) return; + if ( + screenshots.some( + (screenshot) => screenshot.index === -1 || !screenshot.rawUrl, + ) + ) + return; const result: Gql.ChromeExtension = { id, @@ -127,6 +151,7 @@ export async function crawlExtension( longDescription, rating, reviewCount, + screenshots, }; consola.debug("Crawl results:", result); return result; diff --git a/src/rest/getChromeScreenshot.ts b/src/rest/getChromeScreenshot.ts new file mode 100644 index 0000000..aa42a32 --- /dev/null +++ b/src/rest/getChromeScreenshot.ts @@ -0,0 +1,15 @@ +import type { ChromeService } from "../services/chrome-service"; +import { RouteHandler } from "../utils/rest-router"; + +export const getChromeScreenshot = + (chrome: ChromeService): RouteHandler<{ id: string; index: string }> => + async (params) => { + const extension = await chrome.getExtension(params.id); + const index = Number(params.index); + const screenshot = extension?.screenshots.find( + (screenshot) => screenshot.index == index, + ); + + if (screenshot == null) return new Response(null, { status: 404 }); + return Response.redirect(screenshot.rawUrl); + }; diff --git a/src/rest/getFirefoxScreenshot.ts b/src/rest/getFirefoxScreenshot.ts new file mode 100644 index 0000000..6ff3e05 --- /dev/null +++ b/src/rest/getFirefoxScreenshot.ts @@ -0,0 +1,15 @@ +import type { FirefoxService } from "../services/firefox-service"; +import { RouteHandler } from "../utils/rest-router"; + +export const getFirefoxScreenshot = + (firefox: FirefoxService): RouteHandler<{ id: string; index: string }> => + async (params) => { + const addon = await firefox.getAddon(params.id); + const index = Number(params.index); + const screenshot = addon?.screenshots.find( + (screenshot) => screenshot.index == index, + ); + + if (screenshot == null) return new Response(null, { status: 404 }); + return Response.redirect(screenshot.rawUrl); + }; diff --git a/src/schema.gql b/src/schema.gql index 5574bf5..452f26e 100644 --- a/src/schema.gql +++ b/src/schema.gql @@ -33,6 +33,7 @@ type ChromeExtension { lastUpdated: String! rating: Float reviewCount: Int + screenshots: [Screenshot!]! } type FirefoxAddon { @@ -47,4 +48,20 @@ type FirefoxAddon { lastUpdated: String! rating: Float reviewCount: Int + screenshots: [Screenshot!]! +} + +type Screenshot { + """ + The screenshot's order. + """ + index: Int! + """ + The image's raw URL provided by the service. When screenshots are updated, this URL changes. + """ + rawUrl: String! + """ + URL to the image based on the index. If the raw URL changes, the `indexUrl` will remain constant, good for links in README.md files. + """ + indexUrl: String! } diff --git a/src/server.ts b/src/server.ts index db1cf02..3aa9c1d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,10 @@ import playgroundHtmlTemplate from "./public/playground.html"; import consola from "consola"; import { createChromeService } from "./services/chrome-service"; import { createFirefoxService } from "./services/firefox-service"; +import { createRestRouter } from "./utils/rest-router"; +import { getChromeScreenshot } from "./rest/getChromeScreenshot"; +import { getFirefoxScreenshot } from "./rest/getFirefoxScreenshot"; +import { SERVER_ORIGIN } from "./utils/urls"; const playgroundHtml = playgroundHtmlTemplate.replace( "{{VERSION}}", @@ -22,6 +26,16 @@ export function createServer(config?: ServerConfig) { firefox, }); + const restRouter = createRestRouter() + .get( + "/api/rest/chrome-extensions/:id/screenshots/:index", + getChromeScreenshot(chrome), + ) + .get( + "/api/rest/firefox-addons/:id/screenshots/:index", + getFirefoxScreenshot(firefox), + ); + const httpServer = Bun.serve({ port, error(request) { @@ -32,8 +46,15 @@ export function createServer(config?: ServerConfig) { return createResponse(undefined, { status: 204 }); } + const url = new URL(req.url, SERVER_ORIGIN); + + // REST + if (url.pathname.startsWith("/api/rest")) { + return restRouter.fetch(url, req); + } + // GraphQL - if (req.url.endsWith("/api")) { + if (url.pathname.startsWith("/api")) { const data = await graphql.evaluateQuery(req); return createResponse(JSON.stringify(data), { diff --git a/src/services/chrome-service.ts b/src/services/chrome-service.ts index bd0e0f1..8423bdc 100644 --- a/src/services/chrome-service.ts +++ b/src/services/chrome-service.ts @@ -16,8 +16,11 @@ export function createChromeService() { }); return { - getExtension: (id: string) => loader.load(id), - getExtensions: async (ids: string[]) => { + getExtension: (id: string): Promise => + loader.load(id), + getExtensions: async ( + ids: string[], + ): Promise> => { const result = await loader.loadMany(ids); return result.map((item, index) => { if (item instanceof Error) { @@ -29,3 +32,5 @@ export function createChromeService() { }, }; } + +export type ChromeService = ReturnType; diff --git a/src/services/firefox-service.ts b/src/services/firefox-service.ts index 22ecbb1..a284f58 100644 --- a/src/services/firefox-service.ts +++ b/src/services/firefox-service.ts @@ -11,8 +11,11 @@ export function createFirefoxService() { >(HOUR_MS, (ids) => Promise.all(ids.map((id) => firefox.getAddon(id)))); return { - getAddon: (id: string | number) => loader.load(id), - getAddons: async (ids: Array) => { + getAddon: (id: string | number): Promise => + loader.load(id), + getAddons: async ( + ids: Array, + ): Promise> => { const result = await loader.loadMany(ids); return result.map((item) => { if (item == null) return undefined; @@ -25,3 +28,5 @@ export function createFirefoxService() { }, }; } + +export type FirefoxService = ReturnType; diff --git a/src/utils/rest-router.ts b/src/utils/rest-router.ts new file mode 100644 index 0000000..7dc72ac --- /dev/null +++ b/src/utils/rest-router.ts @@ -0,0 +1,42 @@ +import * as radix3 from "radix3"; + +export type RouteHandler = ( + params: TParams, + url: URL, + req: Request, +) => Response | Promise; + +export interface Route { + method: string; + handler: RouteHandler; +} + +export function createRestRouter() { + const r = radix3.createRouter(); + const router = { + get(path: string, handler: RouteHandler) { + r.insert(path, { method: "GET", handler }); + return router; + }, + post(path: string, handler: RouteHandler) { + r.insert(path, { method: "POST", handler }); + return router; + }, + any(path: string, handler: RouteHandler) { + r.insert(path, { method: "ANY", handler }); + return router; + }, + on(method: string, path: string, handler: RouteHandler) { + r.insert(path, { method, handler }); + return router; + }, + async fetch(url: URL, req: Request): Promise { + const match = r.lookup(url.pathname); + if (match && (req.method === match.method || match.method === "ANY")) { + return await match.handler(match.params ?? {}, url, req); + } + return new Response(null, { status: 404 }); + }, + }; + return router; +} diff --git a/src/utils/urls.ts b/src/utils/urls.ts new file mode 100644 index 0000000..46adcff --- /dev/null +++ b/src/utils/urls.ts @@ -0,0 +1,10 @@ +export const SERVER_ORIGIN = + process.env.SERVER_ORIGIN ?? "http://localhost:3000"; + +export function buildScreenshotUrl( + base: "chrome-extensions" | "firefox-addons", + id: string, + index: number, +) { + return `${SERVER_ORIGIN}/api/rest/${base}/${id}/screenshots/${index}`; +}