From 9c9e7302c757dd506195e7d5820548370831b542 Mon Sep 17 00:00:00 2001 From: Daan Klarenbeek Date: Mon, 16 Oct 2023 21:11:40 +0200 Subject: [PATCH] feat(Api): add chunk uploading (#445) * chore: update prisma schemas * feat(Config): expose dataDirectory * feat: add partialFileManager * feat(UploadApi): add chunk upload create api route * fix(ChunkCreateApi): not checking mime-type and extension * feat(UploadApi): add chunk upload api route * feat(UploadApi): add chunk completion api route * feat: add partial file component * feat: add partialFileHandler worker * fix(PartialFileHandler): incorrect id generation * fix(PartialFileWorker): add correct file name generation * fix(UploadChunkCompleteApi): partialFile database entry not deleted * feat(UploadChunkCreateApi): exlude undefined filename from check * fix(UploadChunkUploadApi): incorrect ratelimiting options * feat(Utils): add fileUploader class handler * refactor(CreateDialog): use fileUploader instead of /api/v1/upload api --- .../migration.sql | 13 ++ .../migration.sql | 18 +++ .../migration.sql | 19 +++ .../migration.sql | 19 +++ apps/server/prisma/schema.prisma | 21 ++- apps/server/src/components/Domain.ts | 5 + apps/server/src/components/PartialFile.ts | 144 ++++++++++++++++++ .../src/controllers/PartialFileManager.ts | 46 ++++++ apps/server/src/lib/Config.ts | 16 +- .../routes/api/v1/upload/chunk/complete.ts | 58 +++++++ .../src/routes/api/v1/upload/chunk/create.ts | 98 ++++++++++++ .../src/routes/api/v1/upload/chunk/upload.ts | 55 +++++++ apps/server/src/workers/partialFile.ts | 74 +++++++++ .../src/app/dashboard/files/CreateDialog.tsx | 17 +-- packages/utils/FileUploader.ts | 98 ++++++++++++ packages/utils/package.json | 3 +- packages/utils/tsconfig.json | 3 +- yarn.lock | 3 +- 18 files changed, 684 insertions(+), 26 deletions(-) create mode 100644 apps/server/prisma/migrations/20231016123436_partial_file_added/migration.sql create mode 100644 apps/server/prisma/migrations/20231016123732_partial_file_date_added/migration.sql create mode 100644 apps/server/prisma/migrations/20231016125842_partial_file_domain_relation_added/migration.sql create mode 100644 apps/server/prisma/migrations/20231016132533_partial_file_password_optional/migration.sql create mode 100644 apps/server/src/components/PartialFile.ts create mode 100644 apps/server/src/controllers/PartialFileManager.ts create mode 100644 apps/server/src/routes/api/v1/upload/chunk/complete.ts create mode 100644 apps/server/src/routes/api/v1/upload/chunk/create.ts create mode 100644 apps/server/src/routes/api/v1/upload/chunk/upload.ts create mode 100644 apps/server/src/workers/partialFile.ts create mode 100644 packages/utils/FileUploader.ts diff --git a/apps/server/prisma/migrations/20231016123436_partial_file_added/migration.sql b/apps/server/prisma/migrations/20231016123436_partial_file_added/migration.sql new file mode 100644 index 00000000..4d8560ef --- /dev/null +++ b/apps/server/prisma/migrations/20231016123436_partial_file_added/migration.sql @@ -0,0 +1,13 @@ +-- CreateTable +CREATE TABLE "PartialFile" ( + "id" TEXT NOT NULL, + "path" TEXT NOT NULL, + "domain" TEXT NOT NULL, + "file_name" TEXT NOT NULL, + "mime_type" TEXT NOT NULL, + "visible" BOOLEAN NOT NULL, + "password" TEXT NOT NULL +); + +-- CreateIndex +CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path"); diff --git a/apps/server/prisma/migrations/20231016123732_partial_file_date_added/migration.sql b/apps/server/prisma/migrations/20231016123732_partial_file_date_added/migration.sql new file mode 100644 index 00000000..81619626 --- /dev/null +++ b/apps/server/prisma/migrations/20231016123732_partial_file_date_added/migration.sql @@ -0,0 +1,18 @@ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_PartialFile" ( + "id" TEXT NOT NULL, + "path" TEXT NOT NULL, + "domain" TEXT NOT NULL, + "date" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "file_name" TEXT NOT NULL, + "mime_type" TEXT NOT NULL, + "visible" BOOLEAN NOT NULL, + "password" TEXT NOT NULL +); +INSERT INTO "new_PartialFile" ("domain", "file_name", "id", "mime_type", "password", "path", "visible") SELECT "domain", "file_name", "id", "mime_type", "password", "path", "visible" FROM "PartialFile"; +DROP TABLE "PartialFile"; +ALTER TABLE "new_PartialFile" RENAME TO "PartialFile"; +CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path"); +PRAGMA foreign_key_check; +PRAGMA foreign_keys=ON; diff --git a/apps/server/prisma/migrations/20231016125842_partial_file_domain_relation_added/migration.sql b/apps/server/prisma/migrations/20231016125842_partial_file_domain_relation_added/migration.sql new file mode 100644 index 00000000..e272cc2b --- /dev/null +++ b/apps/server/prisma/migrations/20231016125842_partial_file_domain_relation_added/migration.sql @@ -0,0 +1,19 @@ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_PartialFile" ( + "id" TEXT NOT NULL, + "path" TEXT NOT NULL, + "domain" TEXT NOT NULL, + "date" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "file_name" TEXT NOT NULL, + "mime_type" TEXT NOT NULL, + "visible" BOOLEAN NOT NULL, + "password" TEXT NOT NULL, + CONSTRAINT "PartialFile_domain_fkey" FOREIGN KEY ("domain") REFERENCES "Domain" ("domain") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_PartialFile" ("date", "domain", "file_name", "id", "mime_type", "password", "path", "visible") SELECT "date", "domain", "file_name", "id", "mime_type", "password", "path", "visible" FROM "PartialFile"; +DROP TABLE "PartialFile"; +ALTER TABLE "new_PartialFile" RENAME TO "PartialFile"; +CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path"); +PRAGMA foreign_key_check; +PRAGMA foreign_keys=ON; diff --git a/apps/server/prisma/migrations/20231016132533_partial_file_password_optional/migration.sql b/apps/server/prisma/migrations/20231016132533_partial_file_password_optional/migration.sql new file mode 100644 index 00000000..0c6b9763 --- /dev/null +++ b/apps/server/prisma/migrations/20231016132533_partial_file_password_optional/migration.sql @@ -0,0 +1,19 @@ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_PartialFile" ( + "id" TEXT NOT NULL, + "path" TEXT NOT NULL, + "domain" TEXT NOT NULL, + "date" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "file_name" TEXT NOT NULL, + "mime_type" TEXT NOT NULL, + "visible" BOOLEAN NOT NULL, + "password" TEXT, + CONSTRAINT "PartialFile_domain_fkey" FOREIGN KEY ("domain") REFERENCES "Domain" ("domain") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_PartialFile" ("date", "domain", "file_name", "id", "mime_type", "password", "path", "visible") SELECT "date", "domain", "file_name", "id", "mime_type", "password", "path", "visible" FROM "PartialFile"; +DROP TABLE "PartialFile"; +ALTER TABLE "new_PartialFile" RENAME TO "PartialFile"; +CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path"); +PRAGMA foreign_key_check; +PRAGMA foreign_keys=ON; diff --git a/apps/server/prisma/schema.prisma b/apps/server/prisma/schema.prisma index cb998a5d..9e78b8b2 100644 --- a/apps/server/prisma/schema.prisma +++ b/apps/server/prisma/schema.prisma @@ -39,9 +39,10 @@ model Domain { embedTitle String @default("") embedColor String @default("#000000") - urls Url[] - files File[] - pasteBins Pastebin[] + urls Url[] + files File[] + pasteBins Pastebin[] + PartialFile PartialFile[] } model Token { @@ -110,6 +111,20 @@ model File { @@id([id, domain]) } +model PartialFile { + id String + path String @unique + + domain String + Domain Domain? @relation(fields: [domain], references: [domain]) + date DateTime @default(now()) + + filename String @map("file_name") + mimeType String @map("mime_type") + visible Boolean + password String? +} + model Pastebin { id String path String @unique diff --git a/apps/server/src/components/Domain.ts b/apps/server/src/components/Domain.ts index 39eb948e..c1932d26 100644 --- a/apps/server/src/components/Domain.ts +++ b/apps/server/src/components/Domain.ts @@ -14,6 +14,7 @@ import { PastebinReadScheduler } from "./Domain/PastebinReadScheduler.js"; import { FileViewScheduler } from "./Domain/FIleViewScheduler.js"; import { ShorturlVisitScheduler } from "./Domain/ShorturlVisitScheduler.js"; import type formidable from "formidable"; +import PartialFileManager from "#controllers/PartialFileManager.js"; type iDomain = DomainInterface & { apiTokens: Token[]; @@ -72,6 +73,7 @@ export default class Domain { public embedEnabled!: boolean; public auditlogs: AuditLog; + public partialFileManager!: PartialFileManager; public pastebins = new PastebinReadScheduler(this); public files = new FileViewScheduler(this); @@ -92,6 +94,9 @@ export default class Domain { await this.recordStorage(); await this.syncStorage(); await this.auditlogs.start(); + + const partialFiles = await this.server.prisma.partialFile.findMany({ where: { domain: this.domain } }); + this.partialFileManager = new PartialFileManager(this, partialFiles); } /** Resets this domain and removes all the data from the system */ diff --git a/apps/server/src/components/PartialFile.ts b/apps/server/src/components/PartialFile.ts new file mode 100644 index 00000000..83f2f477 --- /dev/null +++ b/apps/server/src/components/PartialFile.ts @@ -0,0 +1,144 @@ +import { Auth } from "#lib/Auth.js"; +import { join } from "node:path"; +import { PrismaClient, type PartialFile as iPartialFile } from "@prisma/client"; +import Config from "#lib/Config.js"; +import { extension } from "mime-types"; +import { Utils } from "#lib/utils.js"; +import { existsSync, mkdirSync, readdirSync } from "node:fs"; +import type PartialFileManager from "#controllers/PartialFileManager.js"; +import type formidable from "formidable"; +import { rename } from "node:fs/promises"; +import { Worker } from "node:worker_threads"; + +export default class PartialFile { + public readonly manager: PartialFileManager; + + /** The unique identifier for this partial file */ + public readonly id: string; + /** The path to the file chunks */ + public readonly path: string; + + /** The date this partial file was created at */ + public readonly createdAt: Date; + + /** The id of the last chunk */ + public lastChunkId?: string; + /** The list of chunks */ + public chunks: string[] = []; + + public status: "OPEN" | "PROCESSING" | "FINISHED" = "OPEN"; + public documentId: string | undefined; + + /** Self destruction timeout (1 day from creation date) */ + public timeout!: NodeJS.Timeout; + private readonly filename: string; + private readonly mimeType: string; + private readonly password: string | undefined; + private readonly visible: boolean; + + public constructor(manager: PartialFileManager, file: iPartialFile) { + this.manager = manager; + this.id = file.id; + this.path = file.path; + + this.createdAt = file.date; + + this.filename = file.filename; + this.mimeType = file.mimeType; + this.password = file.password ?? undefined; + this.visible = file.visible; + + this.populate(); + this.setTimeout(); + } + + /** + * Register a file chunk + * @param file The file chunk to register + */ + public async registerUpload(file: formidable.File) { + const chunkId = (this.lastChunkId ? Number(this.lastChunkId) + 1 : 0).toString(); + this.lastChunkId = chunkId; + this.chunks.push(chunkId); + + await rename(file.filepath, join(this.path, chunkId)); + } + + public complete() { + if (["PROCESSING", "FINISHED"].includes(this.status)) return; + this.status = "PROCESSING"; + + if (!this.lastChunkId) throw new Error("Missing lastChunkId"); + if (!this.chunks.length) throw new Error("Missing array of chunk ids"); + + const worker = new Worker("./dist/workers/partialFile.js", { + workerData: { + path: this.path, + destination: join(this.manager.domain.filesPath), + id: this.id, + createdAt: this.createdAt, + domain: this.manager.domain.domain, + password: this.password, + filename: this.filename, + mimeType: this.mimeType, + visible: this.visible, + chunks: this.chunks, + lastChunkId: this.lastChunkId + } + }); + + worker.on("message", (documentId) => { + this.documentId = documentId; + this.status = "FINISHED"; + }); + } + + private populate() { + if (!existsSync(this.path)) mkdirSync(this.path, { recursive: true }); + const chunks = readdirSync(this.path).filter((str) => !isNaN(Number(str))); + + // eslint-disable-next-line prefer-destructuring + this.lastChunkId = chunks.sort().reverse()[0]; + this.chunks = chunks; + } + + private setTimeout() { + const timeLeft = this.createdAt.getTime() + 8.64e7 - Date.now(); + const timeout = setTimeout(() => this.manager.delete(this.id), timeLeft); + this.timeout = timeout; + } + + /** + * Creates a new partial file instance + * @param data The partial file configuration + * @param domain The domain associated with this partial file + * @returns + */ + public static async create(data: PartialFileCreateOptions, manager: PartialFileManager) { + const id = Auth.generateToken(32); + const path = join(Config.dataDirectory, "files", "tmp", `chunks_${id}`); + + const fileExt = extension(data.mimeType); + if (!fileExt) throw new Error("Invalid mimetype provided"); + + const name = data.filename + ? data.filename + : Utils.generateId(manager.domain.nameStrategy, manager.domain.nameLength) || Utils.generateId("id", 32)!; + const filename = manager.domain.nameStrategy === "zerowidth" || name.includes(".") ? name : `${name}.${fileExt}`; + + const config = Config.getEnv(); + const password = data.password ? Auth.encryptPassword(data.password, config.encryptionKey) : undefined; + + const client = new PrismaClient(); + const file = await client.partialFile.create({ data: { ...data, id, path, filename, password, domain: manager.domain.domain } }); + + return new PartialFile(manager, file); + } +} + +export interface PartialFileCreateOptions { + filename?: string | undefined; + password?: string | undefined; + mimeType: string; + visible: boolean; +} diff --git a/apps/server/src/controllers/PartialFileManager.ts b/apps/server/src/controllers/PartialFileManager.ts new file mode 100644 index 00000000..35c4d5a1 --- /dev/null +++ b/apps/server/src/controllers/PartialFileManager.ts @@ -0,0 +1,46 @@ +import type Domain from "#components/Domain.js"; +import type { PartialFileCreateOptions } from "#components/PartialFile.js"; +import PartialFile from "#components/PartialFile.js"; +import { Collection } from "@discordjs/collection"; +import type { PartialFile as iPartialFile } from "@prisma/client"; +import { rm } from "node:fs/promises"; + +export default class PartialFileManager { + public readonly domain: Domain; + + /** Collection of active partial file handlers */ + public readonly partials = new Collection(); + + public constructor(domain: Domain, files: iPartialFile[]) { + this.domain = domain; + files.forEach((file) => this.partials.set(file.id, new PartialFile(this, file))); + } + + /** + * Creates a new partial file instance + * @param data The partial file configuration + * @returns + */ + public async create(data: PartialFileCreateOptions) { + const file = await PartialFile.create(data, this); + this.partials.set(file.id, file); + + return file; + } + + /** + * Deletes a partial file handler + * @param id The id of the partial file handler + * @returns + */ + public async delete(id: string) { + const partial = this.partials.get(id); + if (!partial) return; + + await rm(partial.path, { force: true, recursive: true }); + await this.domain.server.prisma.partialFile.delete({ where: { id, path: partial.path } }); + + clearTimeout(partial.timeout); + this.partials.delete(id); + } +} diff --git a/apps/server/src/lib/Config.ts b/apps/server/src/lib/Config.ts index 83b4f320..fcf70f28 100644 --- a/apps/server/src/lib/Config.ts +++ b/apps/server/src/lib/Config.ts @@ -30,19 +30,18 @@ export default class Config { public constructor(public server: Server) {} public async start() { - const dataDir = join(process.cwd(), "..", "..", "data"); - await mkdir(join(dataDir, "logs"), { recursive: true }); - await mkdir(join(dataDir, "files"), { recursive: true }); - await mkdir(join(dataDir, "paste-bins"), { recursive: true }); - await mkdir(join(dataDir, "backups", "archives"), { recursive: true }); - await mkdir(join(dataDir, "backups", "temp"), { recursive: true }); + await mkdir(join(Config.dataDirectory, "logs"), { recursive: true }); + await mkdir(join(Config.dataDirectory, "files"), { recursive: true }); + await mkdir(join(Config.dataDirectory, "paste-bins"), { recursive: true }); + await mkdir(join(Config.dataDirectory, "backups", "archives"), { recursive: true }); + await mkdir(join(Config.dataDirectory, "backups", "temp"), { recursive: true }); // Check if mime-types/extensions are already updated - const flagsRaw = await readFile(join(dataDir, "flags.json"), "utf-8").catch(() => "{}"); + const flagsRaw = await readFile(join(Config.dataDirectory, "flags.json"), "utf-8").catch(() => "{}"); const flags: Record = JSON.parse(flagsRaw); await this.updateFiles(flags["mime-type"], flags.extension); - await writeFile(join(dataDir, "flags.json"), JSON.stringify({ "mime-type": true, extension: true })); + await writeFile(join(Config.dataDirectory, "flags.json"), JSON.stringify({ "mime-type": true, extension: true })); const config = Config.getEnv(); await Config.updateEnv(config); @@ -119,6 +118,7 @@ export default class Config { } public static logger = new Logger({ name: "CONFIG" }); + public static dataDirectory = join(process.cwd(), "..", "..", "data"); public static get VERSION(): string { const file = readFileSync(join(process.cwd(), "..", "..", "package.json"), "utf-8"); diff --git a/apps/server/src/routes/api/v1/upload/chunk/complete.ts b/apps/server/src/routes/api/v1/upload/chunk/complete.ts new file mode 100644 index 00000000..9e231c08 --- /dev/null +++ b/apps/server/src/routes/api/v1/upload/chunk/complete.ts @@ -0,0 +1,58 @@ +import type Domain from "#components/Domain.js"; +import { Utils } from "#lib/utils.js"; +import type Server from "#server.js"; +import { ApplyOptions, Route, methods } from "@snowcrystals/highway"; +import type { NextFunction, Request, Response } from "express"; +import { ZodError, z } from "zod"; + +@ApplyOptions({ ratelimit: { max: 25, windowMs: 1e3 }, middleware: [[methods.POST, "user-api-key"]] }) +export default class ApiRoute extends Route { + public async [methods.POST](req: Request, res: Response, next: NextFunction, { domain }: Record<"domain", Domain>) { + const body = this.parseBody(req.body); + if (body instanceof ZodError) { + const errors = Utils.parseZodError(body); + res.status(400).send({ errors }); + return; + } + + const partialFileHandler = domain.partialFileManager.partials.get(body.id); + if (!partialFileHandler) { + res.status(409).send({ + errors: [{ field: "id", code: "NOT_FOUND", message: "A partialfile handler with the provided id does not exist" }] + }); + return; + } + + if (partialFileHandler.status === "OPEN") { + partialFileHandler.complete(); + res.status(200).send({ status: "PROCESSING", url: null }); + return; + } else if (partialFileHandler.status === "PROCESSING") { + res.status(200).send({ status: "PROCESSING", url: null }); + return; + } + + clearTimeout(partialFileHandler.timeout); + domain.partialFileManager.partials.delete(body.id); + await this.server.prisma.partialFile.delete({ where: { path: partialFileHandler.path } }); + + res.status(200).send({ status: "FINISHED", url: `${req.protocol}://${domain.domain}/files/${partialFileHandler.documentId}` }); + } + + /** + * Parses the request body + * @param body The body to parse + * @returns Parsed json content or ZodError if there was a parsing issue + */ + private parseBody(body: any) { + const schema = z.object({ + id: z.string({ invalid_type_error: "Property 'filename' must be a string", required_error: "Partialfile handler id is required" }) + }); + + try { + return schema.parse(body); + } catch (error) { + return error as ZodError; + } + } +} diff --git a/apps/server/src/routes/api/v1/upload/chunk/create.ts b/apps/server/src/routes/api/v1/upload/chunk/create.ts new file mode 100644 index 00000000..41317bfd --- /dev/null +++ b/apps/server/src/routes/api/v1/upload/chunk/create.ts @@ -0,0 +1,98 @@ +import type Domain from "#components/Domain.js"; +import { Utils } from "#lib/utils.js"; +import type Server from "#server.js"; +import { ApplyOptions, Route, methods } from "@snowcrystals/highway"; +import type { NextFunction, Request, Response } from "express"; +import { ZodError, z } from "zod"; +import { types } from "mime-types"; + +@ApplyOptions({ ratelimit: { max: 1, windowMs: 1e3 }, middleware: [[methods.POST, "user-api-key"]] }) +export default class ApiRoute extends Route { + public async [methods.POST](req: Request, res: Response, next: NextFunction, { domain }: Record<"domain", Domain>) { + const body = this.parseBody(req.body); + if (body instanceof ZodError) { + const errors = Utils.parseZodError(body); + res.status(400).send({ errors }); + return; + } + + switch (domain.extensionsMode) { + case "block": + if (domain.extensions.includes(Utils.getExtension(body.mimeType)!)) { + res.status(403).send({ + errors: [ + { + field: "mimeType", + code: "DISALLOWED_EXTENSION", + message: "The provided mime-type and its file-extension are not allowed." + } + ] + }); + + return; + } + break; + case "pass": + if (!domain.extensions.includes(Utils.getExtension(body.mimeType)!)) { + res.status(403).send({ + errors: [ + { + field: "mimeType", + code: "DISALLOWED_EXTENSION", + message: "The provided mime-type and its file-extension are not allowed." + } + ] + }); + + return; + } + break; + } + + const existingFile = await this.server.prisma.partialFile.findFirst({ where: { filename: body.filename, domain: domain.domain } }); + if (existingFile && body.filename) { + res.status(409).send({ + errors: [{ field: "filename", code: "DUPLICATE_FIELD", message: "A file with the provided filename already exists" }] + }); + return; + } + + try { + const partialFile = await domain.partialFileManager.create(body); + res.status(200).send({ id: partialFile.id }); + } catch (err) { + this.server.logger.fatal("[CHUNK_UPLOADING:CREATE]: Fatal error while creating a partial file handler", err); + res.status(500).send({ + errors: [ + { + field: null, + code: "INTERNAL_SERVER_ERROR", + message: "Unknown error occurred, please try again later." + } + ] + }); + } + } + + /** + * Parses the request body + * @param body The body to parse + * @returns Parsed json content or ZodError if there was a parsing issue + */ + private parseBody(body: any) { + const schema = z.object({ + filename: z.string({ invalid_type_error: "Property 'filename' must be a string" }).optional(), + mimeType: z + .string({ required_error: "A valid mimetype is required", invalid_type_error: "Property 'mimeType' must be a string" }) + .refine((arg) => Object.values(types).includes(arg), "The provided mimeType is not one of mimeType[]"), + visible: z.boolean({ required_error: "Visibility state is required", invalid_type_error: "Property 'visible' must be a boolean" }), + password: z.string({ invalid_type_error: "Property 'password' must be a string" }).optional() + }); + + try { + return schema.parse(body); + } catch (error) { + return error as ZodError; + } + } +} diff --git a/apps/server/src/routes/api/v1/upload/chunk/upload.ts b/apps/server/src/routes/api/v1/upload/chunk/upload.ts new file mode 100644 index 00000000..1f958156 --- /dev/null +++ b/apps/server/src/routes/api/v1/upload/chunk/upload.ts @@ -0,0 +1,55 @@ +import type Domain from "#components/Domain.js"; +import { Auth } from "#lib/Auth.js"; +import type Server from "#server.js"; +import { ApplyOptions, Route, methods } from "@snowcrystals/highway"; +import type { NextFunction, Request, Response } from "express"; +import formidable from "formidable"; + +@ApplyOptions({ ratelimit: { max: 25, windowMs: 1e3 / 2 }, middleware: [[methods.POST, "user-api-key"]] }) +export default class ApiRoute extends Route { + public async [methods.POST](req: Request, res: Response, next: NextFunction, context: Record<"domain", Domain>) { + const { domain } = context; + const getSize = (size: number): number | undefined => { + if (size < 0) return 0; + return size === 0 ? undefined : size; + }; + + const form = formidable({ + multiples: false, + keepExtensions: true, + uploadDir: domain.filesPath, + maxFileSize: getSize(domain.uploadSize), + maxFieldsSize: getSize(domain.maxStorage === 0 ? 0 : domain.maxStorage - domain.storage), + filename: (name, ext) => `${Auth.generateToken(32)}${ext}` + }); + + try { + const [fields, files] = await form.parse(req); + const uploadedFiles = files.file; + if (!uploadedFiles) { + res.status(400).send({ errors: [{ field: "upload", code: "MISSING_UPLOAD_FILES", message: "Missing uploaded files." }] }); + return; + } + + const partialFileId = fields["partial-file-id"]?.[0]; + if (!partialFileId) { + res.status(400).send({ errors: [{ field: "partial-file-id", code: "MISSING_FIELD", message: "Missing partial file id." }] }); + return; + } + + const partialFileHandler = domain.partialFileManager.partials.get(partialFileId); + if (!partialFileHandler) { + res.status(400).send({ errors: [{ field: "partial-file-id", code: "INVALID_FIELD", message: "Invalid partial file id." }] }); + return; + } + + await partialFileHandler.registerUpload(uploadedFiles[0]); + res.sendStatus(204); + } catch (err) { + this.server.logger.fatal(`[CHUNK_UPLOADING:UPLOAD]: Fatal error while uploading a partial file chunk`, err); + res.status(500).send({ + errors: [{ field: null, code: "INTERNAL_ERROR", message: "Internal server error occured, please try again later." }] + }); + } + } +} diff --git a/apps/server/src/workers/partialFile.ts b/apps/server/src/workers/partialFile.ts new file mode 100644 index 00000000..be50b521 --- /dev/null +++ b/apps/server/src/workers/partialFile.ts @@ -0,0 +1,74 @@ +import { Auth } from "#lib/Auth.js"; +import Config from "#lib/Config.js"; +import { Utils } from "#lib/utils.js"; +import { PrismaClient } from "@prisma/client"; +import { Logger } from "@snowcrystals/icicle"; +import { open, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { isMainThread, workerData, parentPort } from "node:worker_threads"; + +interface WorkerData { + path: string; + destination: string; + id: string; + + createdAt: Date; + domain: string; + + password: string | undefined; + filename: string; + mimeType: string; + visible: boolean; + + chunks: string[]; + lastChunkId: string; +} + +const logger = new Logger({ name: "worker::partialFile" }); +const prisma = new PrismaClient(); +const data = workerData as WorkerData; + +if (isMainThread) { + logger.fatal("Instance is run on 'main thread'."); + process.exit(1); +} + +async function run() { + logger.debug("Starting the worker"); + + const fileId = Auth.generateToken(32); + const filePath = join(data.destination, `${fileId}.${Utils.getExtension(data.mimeType)}`); + const stream = await open(filePath, "w"); + + for await (const chunk of data.chunks) { + const buffer = await readFile(join(data.path, chunk)); + await stream.write(buffer, 0, buffer.length); + } + + const stats = await stream.stat(); + await stream.close(); + await rm(data.path, { recursive: true, force: true }); + + const config = Config.getEnv(); + const authBuffer = Buffer.from(`${Auth.generateToken(32)}.${Date.now()}.${data.domain}.${fileId}`).toString("base64"); + const authSecret = Auth.encryptToken(authBuffer, config.encryptionKey); + + await prisma.file.create({ + data: { + id: data.filename, + date: data.createdAt, + path: filePath, + domain: data.domain, + mimeType: data.mimeType, + password: data.password, + visible: data.visible, + size: Utils.parseStorage(stats.size), + authSecret + } + }); + + parentPort?.postMessage(data.filename); + logger.debug("Worker finished"); +} + +void run(); diff --git a/apps/web/src/app/dashboard/files/CreateDialog.tsx b/apps/web/src/app/dashboard/files/CreateDialog.tsx index 1f6d94f1..53b63e6a 100644 --- a/apps/web/src/app/dashboard/files/CreateDialog.tsx +++ b/apps/web/src/app/dashboard/files/CreateDialog.tsx @@ -10,10 +10,11 @@ import React from "react"; import { useForm } from "react-hook-form"; import * as z from "zod"; import { Switch } from "@paperplane/ui/switch"; -import axios, { AxiosError } from "axios"; +import type { AxiosError } from "axios"; import { useToast } from "@paperplane/ui/use-toast"; import Dropzone from "react-dropzone"; import { ApiErrorResponse, formatBytes } from "@paperplane/utils"; +import FileUploader from "@paperplane/utils/FileUploader"; export const CreateDialog: React.FC = () => { const { toast } = useToast(); @@ -31,18 +32,10 @@ export const CreateDialog: React.FC = () => { async function onSubmit(data: z.infer) { try { - const formData = new FormData(); - formData.set("visible", `${data.visible}`); - formData.set("file", data.file); - if (data.name) formData.set("name", data.name); - if (data.password) formData.set("password", data.password); + const uploader = new FileUploader(data.file); + const url = await uploader.upload(data); - const response = await axios.post<{ files: Record; url: string }>("/api/v1/upload", formData, { - withCredentials: true, - headers: { "Content-Type": "multipart/form-data" } - }); - - void navigator.clipboard.writeText(response.data.url); + void navigator.clipboard.writeText(url); toast({ title: "File uploaded", description: "A new file has been created and the url has been copied to your clipboard." }); form.reset({ visible: true }, { keepDirty: false }); } catch (err) { diff --git a/packages/utils/FileUploader.ts b/packages/utils/FileUploader.ts new file mode 100644 index 00000000..365e2848 --- /dev/null +++ b/packages/utils/FileUploader.ts @@ -0,0 +1,98 @@ +import axios from "axios"; + +export default class FileUploader { + public readonly uploadRoute = "/api/v1/upload/chunk/upload"; + public readonly createRoute = "/api/v1/upload/chunk/create"; + public readonly completeRoute = "/api/v1/upload/chunk/complete"; + public readonly chunkSize = 50; // 50.0 MB + + /** The file to upload */ + public readonly file: File; + /** The total amount of chunks for this upload */ + public totalChunks: number; + + public constructor(file: File) { + this.file = file; + this.totalChunks = Math.ceil(this.file.size / (this.chunkSize * 1e3 * 1e3)); + } + + public async upload(opt: UploadOptions) { + if (this.totalChunks === 1) { + const formData = new FormData(); + formData.append("file", this.file); + formData.append("visible", `${opt.visible}`); + + const response = await axios.post("/api/v1/upload", formData, { + headers: { "Content-Type": "multipart/form-data" }, + withCredentials: true + }); + + return response.data.url; + } + + const handlerId = await this.getHandlerId(opt); + for (let i = 0; i < this.totalChunks; i++) { + const chunk = await this.getNextChunk(i); + await this.uploadChunk(chunk, handlerId); + } + + const url = await this.complete(handlerId); + return url; + } + + private async complete(id: string) { + await axios.post(this.completeRoute, { id }, { withCredentials: true }); + let fileUrl: string | null = null; + + while (!fileUrl) { + await new Promise((res) => setTimeout(res, 5e3)); + const response = await axios.post<{ status: string; url: string | null }>(this.completeRoute, { id }, { withCredentials: true }); + fileUrl = response.data.url; + } + + return fileUrl; + } + + private async uploadChunk(chunk: Blob, id: string) { + const formData = new FormData(); + formData.append("file", chunk); + formData.append("partial-file-id", id); + + await axios.post(this.uploadRoute, formData, { + headers: { "Content-Type": "multipart/form-data" }, + withCredentials: true + }); + } + + /** + * Fetches a partialfile handler id + * @param opt The file options + * @returns + */ + private async getHandlerId(opt: UploadOptions) { + const handler = await axios<{ id: string }>(this.createRoute, { + data: { visible: opt.visible, filename: opt.name, password: opt.password, mimeType: this.file.type }, + method: "POST", + withCredentials: true + }); + return handler.data.id; + } + + /** Returns the next file chunk */ + private getNextChunk(index: number) { + return new Promise((res) => { + const length = this.chunkSize * 1e3 * 1e3; + const start = length * index; + + const fileReader = new FileReader(); + fileReader.readAsArrayBuffer(this.file.slice(start, start + length)); + fileReader.onload = () => res(new Blob([fileReader.result!], { type: "application/octet-stream" })); + }); + } +} + +export interface UploadOptions { + visible: boolean; + password?: string | undefined; + name?: string | undefined; +} diff --git a/packages/utils/package.json b/packages/utils/package.json index 603310f5..20a65ae1 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -13,6 +13,7 @@ "typescript": "^5.2.2" }, "dependencies": { - "@sapphire/timestamp": "^1.0.1" + "@sapphire/timestamp": "^1.0.1", + "axios": "1.5.1" } } diff --git a/packages/utils/tsconfig.json b/packages/utils/tsconfig.json index 7840e487..16b3478b 100644 --- a/packages/utils/tsconfig.json +++ b/packages/utils/tsconfig.json @@ -4,7 +4,8 @@ "outDir": "../dist", "module": "ESNext", "target": "ESNext", - "moduleResolution": "node" + "moduleResolution": "node", + "lib": ["dom", "dom.iterable", "esnext"] }, "include": ["**/*.ts", "**/*.tsx", "../../package.json"] } diff --git a/yarn.lock b/yarn.lock index 7370177d..b8768aa2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -459,6 +459,7 @@ __metadata: resolution: "@paperplane/utils@workspace:packages/utils" dependencies: "@sapphire/timestamp": ^1.0.1 + axios: 1.5.1 eslint: 8.51.0 typescript: ^5.2.2 languageName: unknown @@ -2447,7 +2448,7 @@ __metadata: languageName: node linkType: hard -"axios@npm:^1.5.1": +"axios@npm:1.5.1, axios@npm:^1.5.1": version: 1.5.1 resolution: "axios@npm:1.5.1" dependencies: