From 3338836a46e006e19a27e22c67ee278dcb8fddae Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 23 Oct 2024 13:22:15 -0500 Subject: [PATCH 1/5] feat: Extract screenshots from CWS and Firefox addons --- .env | 1 + bun.lockb | Bin 31091 -> 31431 bytes package.json | 3 ++- src/apis/firefox-api.ts | 8 ++++++ src/crawlers/chrome-crawler.ts | 27 +++++++++++++++++++- src/rest/getChromeScreenshot.ts | 15 ++++++++++++ src/schema.gql | 17 +++++++++++++ src/server.ts | 19 +++++++++++++-- src/services/chrome-service.ts | 2 ++ src/utils/rest-router.ts | 42 ++++++++++++++++++++++++++++++++ src/utils/urls.ts | 7 ++++++ 11 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 .env create mode 100644 src/rest/getChromeScreenshot.ts create mode 100644 src/utils/rest-router.ts create mode 100644 src/utils/urls.ts diff --git a/.env b/.env new file mode 100644 index 0000000..9408d04 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +SERVER_ORIGIN=http://localhost:3000 diff --git a/bun.lockb b/bun.lockb index d713d045aeaade0e0d7f1c94112c2213cf74485c..c3fdcd0f1644cbedd0b3247643a4dfdfe4ae7974 100755 GIT binary patch delta 4906 zcmd5=eNa@_6@PbefrqTJqQJTsKIKE)h1~^r5oGZTU7nyQxO^&zSQteupo7U6NG7r3 zj4|sOr#cfgW}2zCar_`NjhX0ZOj;M~sO@ND&6jDWF*U}T)Ed)7`#bNwZJCa3ovDBH z&761M@7{CIy}xtMyZ65L^;hCocf^bK?8B!!KAW>K*im4eW{-R?k`B*}rB zi~7pLHHT|EmiKhEwMnJG>}X~L44{{FwY0W%OVXRbTrNdwju%36{i=@co-U^(-O=%U z6tdk8UC)H(0c4uqaoL9bykk`xQ9dUf0#t+iU)($j8R z*|y<^jz59sR+pf;m$T43#*?(q5HFt?BS|Re56~$?lEEHMpBSbj937{*+Vhrs{)M9_ z0_#_Qx^2wsEh&RpVV@d|`7jq0FuUa+V-UqEGDo__Z^;+#HP{4g2`>??6b$!@VuSBL$d6Zzd(s6 z)R~++B9D2Sq0NgnByZy6=Y3kRdg9OV`@ zG%(UDo~I!07fFux8X_X7Ji0{8qk(9zya_48M8oJ}xLdqKvdJrt!o18<$7D9SbN7CJ$wZeY9J41F=)MLHDF1t&2YbEk>PDtt&<@vE>LAZ31EvDiW8-XZ9Lm zO|->aBKwdY9ASex&S!vlWZ@J*1ct~r#w#0<#4}I^$0oO!LqU{1i!!#;+`kQ^%^MHv zJ`fK?tE)z$am2#N9P5b$;OW)^7Kc=>)T+>IIZ%n(j05%x&=eqz>3cvtNX=!8S?k=$ zgOnEl{iDvfnd`_DStOsqlio4B8<6rk37}a(JYa1c{XpDvgqo6{0XcyToCHP--Zdxt zGb(s0Q$6p&6Z--s`1%-#`v;QUasrkNZasnmBi(X2&}7vYzd7rHihxF{p+2Hp8hN(m zKL9DHi&lNw@hVwuSs++yrSW<>Yc8Ll-7EPWMl>tJoHo%0d7i(sQS53YClV+xPaU>o<8H78YqwBfQ zTwV_0tDJm^u|Bq#r{afeZh@p%TQ1P;hHESh>Ru|*tBFa4?5#n!dKAsPmOumlVHBqiD{HHr<{zw_l-SR-9AA^D0!vQF=&zY5Bm3#>09H3C8^mPq_brwf@xxk0-&@ z{vJKQIC1#;pAw9wB_{npCDrW5l2mjrsi${XI=78*R1U$MyJ0t)VTiL zFV|OjC;#T#kKQ-F+;!M?B)xR;oZ7OyXj|UO>E%@?cYf1sytV$R)3G&U>0ojKbtWq! zjE*JysXfI+6H*jmq>dCnC8V0@98d&VQvGxWXk)4(WC{ZHrkTi=rihWWAt z4R-L$1wSAgW#@t)(AHc<D72vv{HB5*Pz7aA z1wWvzQx!3vt^jQ=0>2_fRM9{Y_!Wa6&>|`<_KWJlK=D~Y7mQ=6u_RyYpg>8i@dK=7 zW5_WxXM6YvIrNEfdhlAw55nYgSmUse^#6{kW?U8RgSMIe7)nldM4hSDUi8u$dSOus zFFW@iXcw3BYR@lC4Fg;u%sWAWS|UmFq1lEP%IOfU<6VKvcva>-hp#FK*YS$W`w!PG zg0S6My#vF^#Z+Fs*n~5my817JZl#0OnHC&H)b)E5G)`ht3^O6%`%oiv6NJC^c$cey@bXp)nE~NFmp5o$^p`?-FJf-I=i&&% zM;w(4@5y-(-n{D}%y~A1Ir$*Gr~Y}dg!4GA&9&f@VPfta6ZRKRw`;6@94(Fpv*&$@ z8RHbJMu#KAdnpd(svABL@GcN4V;y?tx==hq_MsT@Y%xn7IgVsfA%vr1;)ZK%D{b;;3?BA-L}KWhKlF)c;7L%gXhDCfbsFLXJyDlnNEPI%bBcYGQg0E5 z==r(|Yv?ooN5R_f($b2*Hi#uod!8MHR4Z%fx4?tTxr+?fca@;N0QKOgc?tPCUT>k* z^_B2?yxt`a(CzvP{*Wzgn1>Z=YlBOa(ka}_=}tq1HS}wsrumBp-@e%LSG7}@9l?8( zni?(E&@X}!wa)5u;XivB}MiJ{*M_2Q+gWq0?kMe8XbhCZxObhs(O8v5T*eQ58+ z$uS>ahk-+jjOqX%(cd>&tf8L^(z7S_pDFA5C9Lu_s{wMLOU-uJAPPdiC0<+OczxBE zt$WadBZ`Rs`yPAg+!7Z)|HM+4^%UMLyeR#r#uVLII&b?~w0DJioWE2jY1RLZ^jGG- z{fcGi^jKIiJ@j&(b}zM9PvZT;tJ%Nc=~ZvNe`N4A)VooSc|A)P;pGzDfK}+X#pTMf zpWfJZE&rc=-Jqmqi?~aL%{;EoW{V*#hBh~^5GUwXbA<@dv=)~*NARL0RJts0yb!mW&$DdjF%lh9ba<^Z*f9|yYA`j?(OdC>AZK%hTiVg zJ@-o?Vo|>a`VQzTmOhL2F5r%yo_p8e{RD*DZP@W?uy35h>}+j!XYcy1F5v=ZV|BNk43C=!d<1$d^k1Rb`DjxY zh1cKJ*#U>d5IW!w1F*8*4V?t7Teup%@W4+{$d*VlpgtF@utRk?R}Z5p!hZ=w0C0calO=g{oqPoUvQ z_`X%$9jnk=cMtl7esqiwFom1xm@UnA-A;eAm5$nyXmRHJ7bg&aQR4WKHZC^{44 zQ(~foz)XY%g@DR{DuC=X5aW|us6o-}2T|vzW=Fa53BSSWFs#|6#%i*a8t~jr{jr+u zbTpleohemP<21RB8saqN6}u1`K8dGLtWSDq7?vfZx-_|m8eE#~eFy#0HB%g~{Wd(HGjcS}2A* zir)#a0wy>MlM}0g8%7&wDG(DekQ|`?(VFrh>Pn6F9<)!yD&g?j43}$x*hvSwC7+=F zF`Duo>Si155e%6?YJ#TJVSRJEX!4BpI|0o3V+Y;@Vs}gjCSh@L?4k^J8i9D;&Dfm< zs=H|`7$w9^<2#PsJwWWF$@gg>cGB$g10WvU&Q97VjOLYcI-$ae7p^uJ+|Ph+4g3y} z#Uat}#F}S+O)nM#@whR@${Yk@4{f{}l)v0W%q$7N-@Kaxd>@FrG?_gI#2x}EKILN| z?j@Se#QBtQ$R^W`?>y*bHc&ZGoDuNNK-|*KGp{@kqyxE(Xk7qcOG}qTaxIo+BMs8Y~QG?ks3TtjwYw1Nstq`|s zBmhxw)!(iqU7V1{d_<#V{U*(1nj!353%8@n)Fk>x+9YyKaz><&XSkECD6F}47Xz#C zs$k*5nyG(>E2Pip(1T8S;B*L=|3&|ID+EFM{}TE3C6f4!ON1IG^-`usr(}G2F*Dl`mq0;qDbF5_tHzR2UI&nm!qkFN|1gr#Z6xVB~VRPkQQXQ=|Gk)6X_bzRiNf< zT_(}K>>xdx?Iw4Qj-9?CCrI@xfl{aGGL5>Y1!>hZ zHysD^kUBj`$Cm++1B|(xzPS%LPB6EXv6PzdZ2E(`60~1Dyb>%-7{K z+L{l3`QQhXOJxP%R{(wmIu1nVfzAQd7V5H)`U}CY5d46=R8s_gMc`MY%M!W~H34E)M;Swq7>Cx9wv=u)SxGr(^K_yN^Y zSvmNXgI~EW7twj3b3nBfI!;vm72sC^en86xYbxK8l&uvfhLBXSMTa>ba$3guB)pH- zY2|X;-~sJ>QA$6yRs7fCrzmyKDSI(?A~(ILEqiKl^{CjR%Q#~QQMH^NTRyYzs~cwF zD$cE(R+ckBiC`Y5*r+ki6Gj`(mQ@h8aSrD1c;Rze=du{WHqL*X;@P$o!tK^t0~`tn z(gOi^8|p%kc1SuM2xO_8IdLlDQ$C-bnXPDrETW5n3{3p!6-ml;;9STADqNA!$2k_; zxslVXJQpY;-&bCM_BJCsb;aOsa>~$H$519lRFGQbl*hmJz^~pGBq(G-aCPLC6B@muk zFC+(&4e>y-Ad?~KkPOHaNT!8(C*|NzwQ!{UG+=JSlfq%k<54+wIDheh%IJ_g_EBMd z!qO_d@vLyfIr1FsN(j#qM|M7BE`(=d0fc9g*DkN3B@i3|1=hKEA&3XcNQidUPvNKz z4%L@Q+X+Fg#vZo{q$keI6B6I?}2+whp)LfeqxjzTm zF5I}jJ@ijw3~wI7_z3kit0j?pbaZ{u^5c&EzlXg9KENKiSx?Ph@Ws`pu_LhfOz(El z@n$t8a>Z3{?AY7Zdq!H`7Zr&8BwJKDL@6y^d4v|>J(>nuRqPOtwW-MIds@`e$i2K# zK6z^1=X=-U>r!*v$j$shXJg=W#cS2D6c*%p5tb2!F0>?@9UwiXwW=kNTe*1P)n|{+ zTmJ_%E;gNv+_-18YV)Iiv->hEmivYS|K`|kdZkshS>0FDO&vAIQX13HfAz%W6%Ed&zF0W zJ{f|=Z_*whcSuc%-0=TYH}9FhZ#z?bYrr9z6;kEv)D&XpwuMyNyYaL?beDXTMz`0> zW@>Hs;(AT(Ka8#7T!s>Er->c2simW6{Ch}&$N}Mp{3UP8jdy(Yg4N{IM%v%;^xy-X HbI1J)=&V3C diff --git a/package.json b/package.json index e11e4be..eeb0c7c 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "dataloader": "^2.2.2", "graphql": "^16.8.0", "linkedom": "^0.15.3", - "picocolors": "^1.0.0" + "picocolors": "^1.0.0", + "radix3": "^1.1.2" }, "devDependencies": { "@aklinker1/check": "^1.2.0", diff --git a/src/apis/firefox-api.ts b/src/apis/firefox-api.ts index 0a0b2fd..720e06a 100644 --- a/src/apis/firefox-api.ts +++ b/src/apis/firefox-api.ts @@ -1,4 +1,5 @@ import consola from "consola"; +import { buildScreenshotUrl } from "../utils/urls"; export function createFirefoxApiClient() { return { @@ -29,6 +30,13 @@ export function createFirefoxApiClient() { storeUrl: json.url, version: json.current_version.version, dailyActiveUsers: json.average_daily_users, + screenshots: (json.previews as any[]).map( + (preview, i) => ({ + index: i, + rawUrl: preview.image_url, + indexUrl: buildScreenshotUrl("firefox", json.id, i), + }), + ), }; }, }; diff --git a/src/crawlers/chrome-crawler.ts b/src/crawlers/chrome-crawler.ts index bc6c32b..5fb2c20 100644 --- a/src/crawlers/chrome-crawler.ts +++ b/src/crawlers/chrome-crawler.ts @@ -1,5 +1,6 @@ import consola from "consola"; import { HTMLAnchorElement, HTMLElement, parseHTML } from "linkedom"; +import { buildScreenshotUrl } from "../utils/urls"; export async function crawlExtension( id: string, @@ -21,7 +22,7 @@ export async function crawlExtension( const { document } = parseHTML(html); // Uncomment to debug HTML - // Bun.write("chrome.html", document.documentElement.outerHTML); + Bun.write("chrome.html", document.documentElement.outerHTML); // Basic metadata const name = metaContent(document, "property=og:title")?.replace( @@ -106,6 +107,23 @@ export async function crawlExtension( // const rating = extractNumber(ratingDiv.title); // "Average rating: 4.78 stars" // const reviewCount = extractNumber(ratingDiv.textContent); // "(1024)" + //
+ const screenshots = [...document.querySelectorAll("div[data-media-url]")] + .filter((div) => div.getAttribute("data-is-video") === "false") + .map((div) => { + const index = Number(div.getAttribute("data-slide-index") || -1); + return { + index, + rawUrl: div.getAttribute("data-media-url") + "=s1280", // "s1280" gets the full resolution + indexUrl: buildScreenshotUrl("chrome", id, index), + }; + }); + if (name == null) return; if (storeUrl == null) return; if (iconUrl == null) return; @@ -114,6 +132,12 @@ export async function crawlExtension( if (version == null) return; if (shortDescription == null) return; if (longDescription == null) return; + if ( + screenshots.some( + (screenshot) => screenshot.index === -1 || !screenshot.rawUrl, + ) + ) + return; const result: Gql.ChromeExtension = { id, @@ -127,6 +151,7 @@ export async function crawlExtension( longDescription, rating, reviewCount, + screenshots, }; consola.debug("Crawl results:", result); return result; diff --git a/src/rest/getChromeScreenshot.ts b/src/rest/getChromeScreenshot.ts new file mode 100644 index 0000000..aa42a32 --- /dev/null +++ b/src/rest/getChromeScreenshot.ts @@ -0,0 +1,15 @@ +import type { ChromeService } from "../services/chrome-service"; +import { RouteHandler } from "../utils/rest-router"; + +export const getChromeScreenshot = + (chrome: ChromeService): RouteHandler<{ id: string; index: string }> => + async (params) => { + const extension = await chrome.getExtension(params.id); + const index = Number(params.index); + const screenshot = extension?.screenshots.find( + (screenshot) => screenshot.index == index, + ); + + if (screenshot == null) return new Response(null, { status: 404 }); + return Response.redirect(screenshot.rawUrl); + }; diff --git a/src/schema.gql b/src/schema.gql index 5574bf5..452f26e 100644 --- a/src/schema.gql +++ b/src/schema.gql @@ -33,6 +33,7 @@ type ChromeExtension { lastUpdated: String! rating: Float reviewCount: Int + screenshots: [Screenshot!]! } type FirefoxAddon { @@ -47,4 +48,20 @@ type FirefoxAddon { lastUpdated: String! rating: Float reviewCount: Int + screenshots: [Screenshot!]! +} + +type Screenshot { + """ + The screenshot's order. + """ + index: Int! + """ + The image's raw URL provided by the service. When screenshots are updated, this URL changes. + """ + rawUrl: String! + """ + URL to the image based on the index. If the raw URL changes, the `indexUrl` will remain constant, good for links in README.md files. + """ + indexUrl: String! } diff --git a/src/server.ts b/src/server.ts index db1cf02..ead40af 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,8 @@ import playgroundHtmlTemplate from "./public/playground.html"; import consola from "consola"; import { createChromeService } from "./services/chrome-service"; import { createFirefoxService } from "./services/firefox-service"; +import { createRestRouter } from "./utils/rest-router"; +import { getChromeScreenshot } from "./rest/getChromeScreenshot"; const playgroundHtml = playgroundHtmlTemplate.replace( "{{VERSION}}", @@ -22,6 +24,11 @@ export function createServer(config?: ServerConfig) { firefox, }); + const restRouter = createRestRouter().get( + "/api/rest/chrome/:id/screenshots/:index", + getChromeScreenshot(chrome), + ); + const httpServer = Bun.serve({ port, error(request) { @@ -32,8 +39,16 @@ export function createServer(config?: ServerConfig) { return createResponse(undefined, { status: 204 }); } - // GraphQL - if (req.url.endsWith("/api")) { + const url = new URL(req.url, process.env.SERVER_ORIGIN); + + // REST + + if (url.pathname.startsWith("/api/rest")) { + return restRouter.fetch(url, req); + } + + if (url.pathname.startsWith("/api")) { + // GraphQL const data = await graphql.evaluateQuery(req); return createResponse(JSON.stringify(data), { diff --git a/src/services/chrome-service.ts b/src/services/chrome-service.ts index bd0e0f1..9a443ba 100644 --- a/src/services/chrome-service.ts +++ b/src/services/chrome-service.ts @@ -29,3 +29,5 @@ export function createChromeService() { }, }; } + +export type ChromeService = ReturnType; diff --git a/src/utils/rest-router.ts b/src/utils/rest-router.ts new file mode 100644 index 0000000..48301ba --- /dev/null +++ b/src/utils/rest-router.ts @@ -0,0 +1,42 @@ +import * as radix3 from "radix3"; + +export type RouteHandler = ( + params: TParams, + url: URL, + req: Request, +) => Response | Promise; + +export interface Route { + method: string; + handler: RouteHandler; +} + +export function createRestRouter() { + const r = radix3.createRouter(); + const router = { + get(path: string, handler: RouteHandler) { + r.insert(path, { method: "GET", handler }); + return router; + }, + post(path: string, handler: RouteHandler) { + r.insert(path, { method: "POST", handler }); + return router; + }, + any(path: string, handler: RouteHandler) { + r.insert(path, { method: "ANY", handler }); + return router; + }, + on(method: string, path: string, handler: RouteHandler) { + r.insert(path, { method, handler }); + return router; + }, + async fetch(url: URL, req: Request): Promise { + const match = r.lookup(url.pathname); + if (match && (req.method === match.method || match.method === "ANY")) { + return await match.handler(match.params ?? {}, url, req); + } + return new Response(null, { status: 404 }); + }, + }; + return router; +} diff --git a/src/utils/urls.ts b/src/utils/urls.ts new file mode 100644 index 0000000..ddc37b2 --- /dev/null +++ b/src/utils/urls.ts @@ -0,0 +1,7 @@ +export function buildScreenshotUrl( + type: "chrome" | "firefox", + id: string, + index: number, +) { + return `${process.env.SERVER_ORIGIN}/api/rest/${type}/${id}/screenshots/${index}`; +} From 9792c7295f684225d40b557889912fee83111654 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 23 Oct 2024 13:29:32 -0500 Subject: [PATCH 2/5] Cleanup --- .env | 1 - .gitignore | 4 ++++ src/rest/getFirefoxScreenshot.ts | 15 +++++++++++++++ src/server.ts | 14 +++++++++----- src/services/chrome-service.ts | 7 +++++-- src/services/firefox-service.ts | 9 +++++++-- src/utils/rest-router.ts | 8 ++++---- src/utils/urls.ts | 5 ++++- 8 files changed, 48 insertions(+), 15 deletions(-) delete mode 100644 .env create mode 100644 src/rest/getFirefoxScreenshot.ts diff --git a/.env b/.env deleted file mode 100644 index 9408d04..0000000 --- a/.env +++ /dev/null @@ -1 +0,0 @@ -SERVER_ORIGIN=http://localhost:3000 diff --git a/.gitignore b/.gitignore index ae2bb14..afab2ea 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ tsconfig.tsbuildinfo *.njsproj *.sln *.sw? + +# .env files +.env +.env.* diff --git a/src/rest/getFirefoxScreenshot.ts b/src/rest/getFirefoxScreenshot.ts new file mode 100644 index 0000000..6ff3e05 --- /dev/null +++ b/src/rest/getFirefoxScreenshot.ts @@ -0,0 +1,15 @@ +import type { FirefoxService } from "../services/firefox-service"; +import { RouteHandler } from "../utils/rest-router"; + +export const getFirefoxScreenshot = + (firefox: FirefoxService): RouteHandler<{ id: string; index: string }> => + async (params) => { + const addon = await firefox.getAddon(params.id); + const index = Number(params.index); + const screenshot = addon?.screenshots.find( + (screenshot) => screenshot.index == index, + ); + + if (screenshot == null) return new Response(null, { status: 404 }); + return Response.redirect(screenshot.rawUrl); + }; diff --git a/src/server.ts b/src/server.ts index ead40af..7f7d895 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,6 +7,8 @@ import { createChromeService } from "./services/chrome-service"; import { createFirefoxService } from "./services/firefox-service"; import { createRestRouter } from "./utils/rest-router"; import { getChromeScreenshot } from "./rest/getChromeScreenshot"; +import { getFirefoxScreenshot } from "./rest/getFirefoxScreenshot"; +import { SERVER_ORIGIN } from "./utils/urls"; const playgroundHtml = playgroundHtmlTemplate.replace( "{{VERSION}}", @@ -24,10 +26,12 @@ export function createServer(config?: ServerConfig) { firefox, }); - const restRouter = createRestRouter().get( - "/api/rest/chrome/:id/screenshots/:index", - getChromeScreenshot(chrome), - ); + const restRouter = createRestRouter() + .get("/api/rest/chrome/:id/screenshots/:index", getChromeScreenshot(chrome)) + .get( + "/api/rest/firefox/:id/screenshots/:index", + getFirefoxScreenshot(firefox), + ); const httpServer = Bun.serve({ port, @@ -39,7 +43,7 @@ export function createServer(config?: ServerConfig) { return createResponse(undefined, { status: 204 }); } - const url = new URL(req.url, process.env.SERVER_ORIGIN); + const url = new URL(req.url, SERVER_ORIGIN); // REST diff --git a/src/services/chrome-service.ts b/src/services/chrome-service.ts index 9a443ba..8423bdc 100644 --- a/src/services/chrome-service.ts +++ b/src/services/chrome-service.ts @@ -16,8 +16,11 @@ export function createChromeService() { }); return { - getExtension: (id: string) => loader.load(id), - getExtensions: async (ids: string[]) => { + getExtension: (id: string): Promise => + loader.load(id), + getExtensions: async ( + ids: string[], + ): Promise> => { const result = await loader.loadMany(ids); return result.map((item, index) => { if (item instanceof Error) { diff --git a/src/services/firefox-service.ts b/src/services/firefox-service.ts index 22ecbb1..a284f58 100644 --- a/src/services/firefox-service.ts +++ b/src/services/firefox-service.ts @@ -11,8 +11,11 @@ export function createFirefoxService() { >(HOUR_MS, (ids) => Promise.all(ids.map((id) => firefox.getAddon(id)))); return { - getAddon: (id: string | number) => loader.load(id), - getAddons: async (ids: Array) => { + getAddon: (id: string | number): Promise => + loader.load(id), + getAddons: async ( + ids: Array, + ): Promise> => { const result = await loader.loadMany(ids); return result.map((item) => { if (item == null) return undefined; @@ -25,3 +28,5 @@ export function createFirefoxService() { }, }; } + +export type FirefoxService = ReturnType; diff --git a/src/utils/rest-router.ts b/src/utils/rest-router.ts index 48301ba..7dc72ac 100644 --- a/src/utils/rest-router.ts +++ b/src/utils/rest-router.ts @@ -14,19 +14,19 @@ export interface Route { export function createRestRouter() { const r = radix3.createRouter(); const router = { - get(path: string, handler: RouteHandler) { + get(path: string, handler: RouteHandler) { r.insert(path, { method: "GET", handler }); return router; }, - post(path: string, handler: RouteHandler) { + post(path: string, handler: RouteHandler) { r.insert(path, { method: "POST", handler }); return router; }, - any(path: string, handler: RouteHandler) { + any(path: string, handler: RouteHandler) { r.insert(path, { method: "ANY", handler }); return router; }, - on(method: string, path: string, handler: RouteHandler) { + on(method: string, path: string, handler: RouteHandler) { r.insert(path, { method, handler }); return router; }, diff --git a/src/utils/urls.ts b/src/utils/urls.ts index ddc37b2..547f71c 100644 --- a/src/utils/urls.ts +++ b/src/utils/urls.ts @@ -1,7 +1,10 @@ +export const SERVER_ORIGIN = + process.env.SERVER_ORIGIN ?? "http://localhost:3000"; + export function buildScreenshotUrl( type: "chrome" | "firefox", id: string, index: number, ) { - return `${process.env.SERVER_ORIGIN}/api/rest/${type}/${id}/screenshots/${index}`; + return `${SERVER_ORIGIN}/api/rest/${type}/${id}/screenshots/${index}`; } From 1659cec4d764784a59099d25f0754a3e05052ba0 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 23 Oct 2024 13:30:45 -0500 Subject: [PATCH 3/5] Cleanup --- src/server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 7f7d895..8c1181d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -46,7 +46,6 @@ export function createServer(config?: ServerConfig) { const url = new URL(req.url, SERVER_ORIGIN); // REST - if (url.pathname.startsWith("/api/rest")) { return restRouter.fetch(url, req); } From bb911ec3ddd0bc3a7d6c938889ee4251fbdcc5fd Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 23 Oct 2024 13:31:17 -0500 Subject: [PATCH 4/5] Clenaup --- src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 8c1181d..818c073 100644 --- a/src/server.ts +++ b/src/server.ts @@ -50,8 +50,8 @@ export function createServer(config?: ServerConfig) { return restRouter.fetch(url, req); } + // GraphQL if (url.pathname.startsWith("/api")) { - // GraphQL const data = await graphql.evaluateQuery(req); return createResponse(JSON.stringify(data), { From b1f14afa35665d8055d005e8a03668997f19372a Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 23 Oct 2024 13:36:22 -0500 Subject: [PATCH 5/5] Fix testsg --- src/apis/firefox-api.ts | 2 +- src/crawlers/__tests__/chrome-crawler.test.ts | 14 ++++++++++++++ src/crawlers/chrome-crawler.ts | 2 +- src/server.ts | 7 +++++-- src/utils/urls.ts | 4 ++-- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/apis/firefox-api.ts b/src/apis/firefox-api.ts index 720e06a..571339f 100644 --- a/src/apis/firefox-api.ts +++ b/src/apis/firefox-api.ts @@ -34,7 +34,7 @@ export function createFirefoxApiClient() { (preview, i) => ({ index: i, rawUrl: preview.image_url, - indexUrl: buildScreenshotUrl("firefox", json.id, i), + indexUrl: buildScreenshotUrl("firefox-addons", json.id, i), }), ), }; diff --git a/src/crawlers/__tests__/chrome-crawler.test.ts b/src/crawlers/__tests__/chrome-crawler.test.ts index a840bab..d973344 100644 --- a/src/crawlers/__tests__/chrome-crawler.test.ts +++ b/src/crawlers/__tests__/chrome-crawler.test.ts @@ -22,6 +22,20 @@ describe("Chrome Web Store Crawler", () => { "https://chromewebstore.google.com/detail/github-better-line-counts/ocfdgncpifmegplaglcnglhioflaimkd", version: expect.any(String), weeklyActiveUsers: expect.any(Number), + screenshots: [ + { + index: 0, + indexUrl: + "http://localhost:3000/api/rest/chrome-extensions/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/0", + rawUrl: expect.any(String), + }, + { + index: 1, + indexUrl: + "http://localhost:3000/api/rest/chrome-extensions/ocfdgncpifmegplaglcnglhioflaimkd/screenshots/1", + rawUrl: expect.any(String), + }, + ], }); }); }); diff --git a/src/crawlers/chrome-crawler.ts b/src/crawlers/chrome-crawler.ts index 5fb2c20..59e451e 100644 --- a/src/crawlers/chrome-crawler.ts +++ b/src/crawlers/chrome-crawler.ts @@ -120,7 +120,7 @@ export async function crawlExtension( return { index, rawUrl: div.getAttribute("data-media-url") + "=s1280", // "s1280" gets the full resolution - indexUrl: buildScreenshotUrl("chrome", id, index), + indexUrl: buildScreenshotUrl("chrome-extensions", id, index), }; }); diff --git a/src/server.ts b/src/server.ts index 818c073..3aa9c1d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -27,9 +27,12 @@ export function createServer(config?: ServerConfig) { }); const restRouter = createRestRouter() - .get("/api/rest/chrome/:id/screenshots/:index", getChromeScreenshot(chrome)) .get( - "/api/rest/firefox/:id/screenshots/:index", + "/api/rest/chrome-extensions/:id/screenshots/:index", + getChromeScreenshot(chrome), + ) + .get( + "/api/rest/firefox-addons/:id/screenshots/:index", getFirefoxScreenshot(firefox), ); diff --git a/src/utils/urls.ts b/src/utils/urls.ts index 547f71c..46adcff 100644 --- a/src/utils/urls.ts +++ b/src/utils/urls.ts @@ -2,9 +2,9 @@ export const SERVER_ORIGIN = process.env.SERVER_ORIGIN ?? "http://localhost:3000"; export function buildScreenshotUrl( - type: "chrome" | "firefox", + base: "chrome-extensions" | "firefox-addons", id: string, index: number, ) { - return `${SERVER_ORIGIN}/api/rest/${type}/${id}/screenshots/${index}`; + return `${SERVER_ORIGIN}/api/rest/${base}/${id}/screenshots/${index}`; }