From b31227331710626c9aa90b39453d2ddcea9b6764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Thu, 24 Oct 2024 10:02:31 +0200 Subject: [PATCH 1/7] Cancels downloading when reaching 10 MB --- src/app/api/proxy/route.ts | 46 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 22226634..17a25256 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -17,6 +17,48 @@ export async function GET(req: NextRequest) { } catch { return makeAPIErrorResponse(400, "Invalid \"url\" query parameter.") } - const file = await fetch(url).then(r => r.blob()) - return new NextResponse(file, { status: 200 }) + try { + const maxBytes = 10 * 1024 * 1024 // 10 MB + const file = await downloadFile({ url, maxBytes }) + console.log("Downloaded") + return new NextResponse(file, { status: 200 }) + } catch (error) { + if (error instanceof Error == false) { + return makeAPIErrorResponse(500, "An unknown error occurred.") + } + if (error.name === "AbortError") { + return makeAPIErrorResponse(413, "The operation was aborted.") + } else if (error.name === "TimeoutError") { + return makeAPIErrorResponse(408, "The operation timed out.") + } else { + return makeAPIErrorResponse(500, error.message) + } + } +} + +async function downloadFile(params: { url: URL, maxBytes: number }): Promise { + const { url, maxBytes } = params + const abortController = new AbortController() + const response = await fetch(url, { + signal: AbortSignal.any([abortController.signal]) + }) + if (!response.body) { + throw new Error("Response body unavailable") + } + let totalBytes = 0 + const reader = response.body.getReader() + // eslint-disable-next-line no-constant-condition + while (true) { + // eslint-disable-next-line no-await-in-loop + const { done, value } = await reader.read() + if (done) { + break + } + totalBytes += value.length + if (totalBytes >= maxBytes) { + abortController.abort() + break + } + } + return await response.blob() } From 334b347ac73f7dc6b4e8f3f7565081be185d5564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Thu, 24 Oct 2024 10:02:41 +0200 Subject: [PATCH 2/7] Timeout downloading after 30 seconds --- src/app/api/proxy/route.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 17a25256..537000f5 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -39,8 +39,9 @@ export async function GET(req: NextRequest) { async function downloadFile(params: { url: URL, maxBytes: number }): Promise { const { url, maxBytes } = params const abortController = new AbortController() + const timeoutSignal = AbortSignal.timeout(30 * 1000) const response = await fetch(url, { - signal: AbortSignal.any([abortController.signal]) + signal: AbortSignal.any([abortController.signal, timeoutSignal]) }) if (!response.body) { throw new Error("Response body unavailable") From 62f0912b56c0bc67a23e64e22b649eb26707c0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Thu, 24 Oct 2024 10:09:16 +0200 Subject: [PATCH 3/7] Removes debug log --- src/app/api/proxy/route.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 537000f5..7f5ae278 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -20,7 +20,6 @@ export async function GET(req: NextRequest) { try { const maxBytes = 10 * 1024 * 1024 // 10 MB const file = await downloadFile({ url, maxBytes }) - console.log("Downloaded") return new NextResponse(file, { status: 200 }) } catch (error) { if (error instanceof Error == false) { From 884500148669bc17bb3e3844528e782eb83e6202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Thu, 24 Oct 2024 10:20:25 +0200 Subject: [PATCH 4/7] Makes max file size and timeout configurable --- .env.example | 2 ++ src/app/api/proxy/route.ts | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index d69da90c..1e98a941 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,8 @@ POSTGRESQL_DB=framna-docs REPOSITORY_NAME_SUFFIX=-openapi HIDDEN_REPOSITORIES= NEW_PROJECT_TEMPLATE_REPOSITORY=shapehq/starter-openapi +PROXY_API_MAXIMUM_FILE_SIZE_IN_MEGABYTES = 10 +PROXY_API_TIMEOUT_IN_SECONDS = 30 GITHUB_WEBHOOK_SECRET=preshared secret also put in app configuration in GitHub GITHUB_WEBHOK_REPOSITORY_ALLOWLIST= GITHUB_WEBHOK_REPOSITORY_DISALLOWLIST= diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 7f5ae278..0af1c4cd 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from "next/server" import { makeAPIErrorResponse, makeUnauthenticatedAPIErrorResponse } from "@/common" import { session } from "@/composition" +import { env } from "@/common" export async function GET(req: NextRequest) { const isAuthenticated = await session.getIsAuthenticated() @@ -18,8 +19,10 @@ export async function GET(req: NextRequest) { return makeAPIErrorResponse(400, "Invalid \"url\" query parameter.") } try { - const maxBytes = 10 * 1024 * 1024 // 10 MB - const file = await downloadFile({ url, maxBytes }) + const maxMegabytes = Number(env.getOrThrow("PROXY_API_MAXIMUM_FILE_SIZE_IN_MEGABYTES")) + const timeoutInSeconds = Number(env.getOrThrow("PROXY_API_TIMEOUT_IN_SECONDS")) + const maxBytes = maxMegabytes * 1024 * 1024 + const file = await downloadFile({ url, maxBytes, timeoutInSeconds }) return new NextResponse(file, { status: 200 }) } catch (error) { if (error instanceof Error == false) { @@ -35,10 +38,14 @@ export async function GET(req: NextRequest) { } } -async function downloadFile(params: { url: URL, maxBytes: number }): Promise { - const { url, maxBytes } = params +async function downloadFile(params: { + url: URL, + maxBytes: number, + timeoutInSeconds: number +}): Promise { + const { url, maxBytes, timeoutInSeconds } = params const abortController = new AbortController() - const timeoutSignal = AbortSignal.timeout(30 * 1000) + const timeoutSignal = AbortSignal.timeout(timeoutInSeconds * 1000) const response = await fetch(url, { signal: AbortSignal.any([abortController.signal, timeoutSignal]) }) From d6dbfb3fe7ab525cace39a0cf3a5f8da2a1bfe4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Thu, 24 Oct 2024 10:20:54 +0200 Subject: [PATCH 5/7] Fixes blob not returned correctly --- src/app/api/proxy/route.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 0af1c4cd..5d6b8c02 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -3,6 +3,11 @@ import { makeAPIErrorResponse, makeUnauthenticatedAPIErrorResponse } from "@/com import { session } from "@/composition" import { env } from "@/common" +const ErrorName = { + MAX_FILE_SIZE_EXCEEDED: "MaxFileSizeExceededError", + TIMEOUT: "TimeoutError" +} + export async function GET(req: NextRequest) { const isAuthenticated = await session.getIsAuthenticated() if (!isAuthenticated) { @@ -25,12 +30,13 @@ export async function GET(req: NextRequest) { const file = await downloadFile({ url, maxBytes, timeoutInSeconds }) return new NextResponse(file, { status: 200 }) } catch (error) { + console.log(error) if (error instanceof Error == false) { return makeAPIErrorResponse(500, "An unknown error occurred.") } - if (error.name === "AbortError") { + if (error.name === ErrorName.MAX_FILE_SIZE_EXCEEDED) { return makeAPIErrorResponse(413, "The operation was aborted.") - } else if (error.name === "TimeoutError") { + } else if (error.name === ErrorName.TIMEOUT) { return makeAPIErrorResponse(408, "The operation timed out.") } else { return makeAPIErrorResponse(500, error.message) @@ -53,7 +59,9 @@ async function downloadFile(params: { throw new Error("Response body unavailable") } let totalBytes = 0 + let didExceedMaxBytes = false const reader = response.body.getReader() + let chunks: Uint8Array[] = [] // eslint-disable-next-line no-constant-condition while (true) { // eslint-disable-next-line no-await-in-loop @@ -62,10 +70,17 @@ async function downloadFile(params: { break } totalBytes += value.length + chunks.push(value) if (totalBytes >= maxBytes) { + didExceedMaxBytes = true abortController.abort() break } } - return await response.blob() + if (didExceedMaxBytes) { + const error = new Error("Maximum file size exceeded") + error.name = ErrorName.MAX_FILE_SIZE_EXCEEDED + throw error + } + return new Blob(chunks) } From f1b540f017b62779257d2a9ce1d3e96514ed0115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Thu, 24 Oct 2024 11:37:40 +0200 Subject: [PATCH 6/7] Removes error logging --- src/app/api/proxy/route.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 5d6b8c02..413c11b6 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -30,7 +30,6 @@ export async function GET(req: NextRequest) { const file = await downloadFile({ url, maxBytes, timeoutInSeconds }) return new NextResponse(file, { status: 200 }) } catch (error) { - console.log(error) if (error instanceof Error == false) { return makeAPIErrorResponse(500, "An unknown error occurred.") } From f1f9296f0e1ed6c4cfb6eccdda0e3fb257afc42a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Thu, 24 Oct 2024 11:43:21 +0200 Subject: [PATCH 7/7] Fixes linting errors --- src/app/api/proxy/route.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 413c11b6..7b96c170 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -1,7 +1,6 @@ import { NextRequest, NextResponse } from "next/server" -import { makeAPIErrorResponse, makeUnauthenticatedAPIErrorResponse } from "@/common" +import { env, makeAPIErrorResponse, makeUnauthenticatedAPIErrorResponse } from "@/common" import { session } from "@/composition" -import { env } from "@/common" const ErrorName = { MAX_FILE_SIZE_EXCEEDED: "MaxFileSizeExceededError", @@ -60,7 +59,7 @@ async function downloadFile(params: { let totalBytes = 0 let didExceedMaxBytes = false const reader = response.body.getReader() - let chunks: Uint8Array[] = [] + const chunks: Uint8Array[] = [] // eslint-disable-next-line no-constant-condition while (true) { // eslint-disable-next-line no-await-in-loop