feat(Api): add chunk uploading (#445)

* chore: update prisma schemas

* feat(Config): expose dataDirectory

* feat: add partialFileManager

* feat(UploadApi): add chunk upload create api route

* fix(ChunkCreateApi): not checking mime-type and extension

* feat(UploadApi): add chunk upload api route

* feat(UploadApi): add chunk completion api route

* feat: add partial file component

* feat: add partialFileHandler worker

* fix(PartialFileHandler): incorrect id generation

* fix(PartialFileWorker): add correct file name generation

* fix(UploadChunkCompleteApi): partialFile database entry not deleted

* feat(UploadChunkCreateApi): exlude undefined filename from check

* fix(UploadChunkUploadApi): incorrect ratelimiting options

* feat(Utils): add fileUploader class handler

* refactor(CreateDialog): use fileUploader instead of /api/v1/upload api

apps/server/prisma/migrations/20231016123436_partial_file_added/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "PartialFile" (
+    "id" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "domain" TEXT NOT NULL,
+    "file_name" TEXT NOT NULL,
+    "mime_type" TEXT NOT NULL,
+    "visible" BOOLEAN NOT NULL,
+    "password" TEXT NOT NULL
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path");

apps/server/prisma/migrations/20231016123732_partial_file_date_added/migration.sql

@@ -0,0 +1,18 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_PartialFile" (
+    "id" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "domain" TEXT NOT NULL,
+    "date" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "file_name" TEXT NOT NULL,
+    "mime_type" TEXT NOT NULL,
+    "visible" BOOLEAN NOT NULL,
+    "password" TEXT NOT NULL
+);
+INSERT INTO "new_PartialFile" ("domain", "file_name", "id", "mime_type", "password", "path", "visible") SELECT "domain", "file_name", "id", "mime_type", "password", "path", "visible" FROM "PartialFile";
+DROP TABLE "PartialFile";
+ALTER TABLE "new_PartialFile" RENAME TO "PartialFile";
+CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/server/prisma/migrations/20231016125842_partial_file_domain_relation_added/migration.sql

@@ -0,0 +1,19 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_PartialFile" (
+    "id" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "domain" TEXT NOT NULL,
+    "date" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "file_name" TEXT NOT NULL,
+    "mime_type" TEXT NOT NULL,
+    "visible" BOOLEAN NOT NULL,
+    "password" TEXT NOT NULL,
+    CONSTRAINT "PartialFile_domain_fkey" FOREIGN KEY ("domain") REFERENCES "Domain" ("domain") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_PartialFile" ("date", "domain", "file_name", "id", "mime_type", "password", "path", "visible") SELECT "date", "domain", "file_name", "id", "mime_type", "password", "path", "visible" FROM "PartialFile";
+DROP TABLE "PartialFile";
+ALTER TABLE "new_PartialFile" RENAME TO "PartialFile";
+CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/server/prisma/migrations/20231016132533_partial_file_password_optional/migration.sql

@@ -0,0 +1,19 @@
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_PartialFile" (
+    "id" TEXT NOT NULL,
+    "path" TEXT NOT NULL,
+    "domain" TEXT NOT NULL,
+    "date" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "file_name" TEXT NOT NULL,
+    "mime_type" TEXT NOT NULL,
+    "visible" BOOLEAN NOT NULL,
+    "password" TEXT,
+    CONSTRAINT "PartialFile_domain_fkey" FOREIGN KEY ("domain") REFERENCES "Domain" ("domain") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_PartialFile" ("date", "domain", "file_name", "id", "mime_type", "password", "path", "visible") SELECT "date", "domain", "file_name", "id", "mime_type", "password", "path", "visible" FROM "PartialFile";
+DROP TABLE "PartialFile";
+ALTER TABLE "new_PartialFile" RENAME TO "PartialFile";
+CREATE UNIQUE INDEX "PartialFile_path_key" ON "PartialFile"("path");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

apps/server/prisma/schema.prisma

@@ -39,9 +39,10 @@ model Domain {
   embedTitle       String  @default("")
   embedColor       String  @default("#000000")
 
-  urls      Url[]
-  files     File[]
-  pasteBins Pastebin[]
+  urls        Url[]
+  files       File[]
+  pasteBins   Pastebin[]
+  PartialFile PartialFile[]
 }
 
 model Token {
@@ -110,6 +111,20 @@ model File {
   @@id([id, domain])
 }
 
+model PartialFile {
+  id   String
+  path String @unique
+
+  domain String
+  Domain Domain?  @relation(fields: [domain], references: [domain])
+  date   DateTime @default(now())
+
+  filename String  @map("file_name")
+  mimeType String  @map("mime_type")
+  visible  Boolean
+  password String?
+}
+
 model Pastebin {
   id   String
   path String @unique

apps/server/src/components/Domain.ts

@@ -14,6 +14,7 @@ import { PastebinReadScheduler } from "./Domain/PastebinReadScheduler.js";
 import { FileViewScheduler } from "./Domain/FIleViewScheduler.js";
 import { ShorturlVisitScheduler } from "./Domain/ShorturlVisitScheduler.js";
 import type formidable from "formidable";
+import PartialFileManager from "#controllers/PartialFileManager.js";
 
 type iDomain = DomainInterface & {
 	apiTokens: Token[];
@@ -72,6 +73,7 @@ export default class Domain {
 	public embedEnabled!: boolean;
 
 	public auditlogs: AuditLog;
+	public partialFileManager!: PartialFileManager;
 
 	public pastebins = new PastebinReadScheduler(this);
 	public files = new FileViewScheduler(this);
@@ -92,6 +94,9 @@ export default class Domain {
 		await this.recordStorage();
 		await this.syncStorage();
 		await this.auditlogs.start();
+
+		const partialFiles = await this.server.prisma.partialFile.findMany({ where: { domain: this.domain } });
+		this.partialFileManager = new PartialFileManager(this, partialFiles);
 	}
 
 	/** Resets this domain and removes all the data from the system */

apps/server/src/components/PartialFile.ts

@@ -0,0 +1,144 @@
+import { Auth } from "#lib/Auth.js";
+import { join } from "node:path";
+import { PrismaClient, type PartialFile as iPartialFile } from "@prisma/client";
+import Config from "#lib/Config.js";
+import { extension } from "mime-types";
+import { Utils } from "#lib/utils.js";
+import { existsSync, mkdirSync, readdirSync } from "node:fs";
+import type PartialFileManager from "#controllers/PartialFileManager.js";
+import type formidable from "formidable";
+import { rename } from "node:fs/promises";
+import { Worker } from "node:worker_threads";
+
+export default class PartialFile {
+	public readonly manager: PartialFileManager;
+
+	/** The unique identifier for this partial file */
+	public readonly id: string;
+	/** The path to the file chunks */
+	public readonly path: string;
+
+	/** The date this partial file was created at */
+	public readonly createdAt: Date;
+
+	/** The id of the last chunk */
+	public lastChunkId?: string;
+	/** The list of chunks */
+	public chunks: string[] = [];
+
+	public status: "OPEN" | "PROCESSING" | "FINISHED" = "OPEN";
+	public documentId: string | undefined;
+
+	/** Self destruction timeout (1 day from creation date) */
+	public timeout!: NodeJS.Timeout;
+	private readonly filename: string;
+	private readonly mimeType: string;
+	private readonly password: string | undefined;
+	private readonly visible: boolean;
+
+	public constructor(manager: PartialFileManager, file: iPartialFile) {
+		this.manager = manager;
+		this.id = file.id;
+		this.path = file.path;
+
+		this.createdAt = file.date;
+
+		this.filename = file.filename;
+		this.mimeType = file.mimeType;
+		this.password = file.password ?? undefined;
+		this.visible = file.visible;
+
+		this.populate();
+		this.setTimeout();
+	}
+
+	/**
+	 * Register a file chunk
+	 * @param file The file chunk to register
+	 */
+	public async registerUpload(file: formidable.File) {
+		const chunkId = (this.lastChunkId ? Number(this.lastChunkId) + 1 : 0).toString();
+		this.lastChunkId = chunkId;
+		this.chunks.push(chunkId);
+
+		await rename(file.filepath, join(this.path, chunkId));
+	}
+
+	public complete() {
+		if (["PROCESSING", "FINISHED"].includes(this.status)) return;
+		this.status = "PROCESSING";
+
+		if (!this.lastChunkId) throw new Error("Missing lastChunkId");
+		if (!this.chunks.length) throw new Error("Missing array of chunk ids");
+
+		const worker = new Worker("./dist/workers/partialFile.js", {
+			workerData: {
+				path: this.path,
+				destination: join(this.manager.domain.filesPath),
+				id: this.id,
+				createdAt: this.createdAt,
+				domain: this.manager.domain.domain,
+				password: this.password,
+				filename: this.filename,
+				mimeType: this.mimeType,
+				visible: this.visible,
+				chunks: this.chunks,
+				lastChunkId: this.lastChunkId
+			}
+		});
+
+		worker.on("message", (documentId) => {
+			this.documentId = documentId;
+			this.status = "FINISHED";
+		});
+	}
+
+	private populate() {
+		if (!existsSync(this.path)) mkdirSync(this.path, { recursive: true });
+		const chunks = readdirSync(this.path).filter((str) => !isNaN(Number(str)));
+
+		// eslint-disable-next-line prefer-destructuring
+		this.lastChunkId = chunks.sort().reverse()[0];
+		this.chunks = chunks;
+	}
+
+	private setTimeout() {
+		const timeLeft = this.createdAt.getTime() + 8.64e7 - Date.now();
+		const timeout = setTimeout(() => this.manager.delete(this.id), timeLeft);
+		this.timeout = timeout;
+	}
+
+	/**
+	 * Creates a new partial file instance
+	 * @param data The partial file configuration
+	 * @param domain The domain associated with this partial file
+	 * @returns
+	 */
+	public static async create(data: PartialFileCreateOptions, manager: PartialFileManager) {
+		const id = Auth.generateToken(32);
+		const path = join(Config.dataDirectory, "files", "tmp", `chunks_${id}`);
+
+		const fileExt = extension(data.mimeType);
+		if (!fileExt) throw new Error("Invalid mimetype provided");
+
+		const name = data.filename
+			? data.filename
+			: Utils.generateId(manager.domain.nameStrategy, manager.domain.nameLength) || Utils.generateId("id", 32)!;
+		const filename = manager.domain.nameStrategy === "zerowidth" || name.includes(".") ? name : `${name}.${fileExt}`;
+
+		const config = Config.getEnv();
+		const password = data.password ? Auth.encryptPassword(data.password, config.encryptionKey) : undefined;
+
+		const client = new PrismaClient();
+		const file = await client.partialFile.create({ data: { ...data, id, path, filename, password, domain: manager.domain.domain } });
+
+		return new PartialFile(manager, file);
+	}
+}
+
+export interface PartialFileCreateOptions {
+	filename?: string | undefined;
+	password?: string | undefined;
+	mimeType: string;
+	visible: boolean;
+}

apps/server/src/controllers/PartialFileManager.ts

@@ -0,0 +1,46 @@
+import type Domain from "#components/Domain.js";
+import type { PartialFileCreateOptions } from "#components/PartialFile.js";
+import PartialFile from "#components/PartialFile.js";
+import { Collection } from "@discordjs/collection";
+import type { PartialFile as iPartialFile } from "@prisma/client";
+import { rm } from "node:fs/promises";
+
+export default class PartialFileManager {
+	public readonly domain: Domain;
+
+	/** Collection of active partial file handlers */
+	public readonly partials = new Collection<string, PartialFile>();
+
+	public constructor(domain: Domain, files: iPartialFile[]) {
+		this.domain = domain;
+		files.forEach((file) => this.partials.set(file.id, new PartialFile(this, file)));
+	}
+
+	/**
+	 * Creates a new partial file instance
+	 * @param data The partial file configuration
+	 * @returns
+	 */
+	public async create(data: PartialFileCreateOptions) {
+		const file = await PartialFile.create(data, this);
+		this.partials.set(file.id, file);
+
+		return file;
+	}
+
+	/**
+	 * Deletes a partial file handler
+	 * @param id The id of the partial file handler
+	 * @returns
+	 */
+	public async delete(id: string) {
+		const partial = this.partials.get(id);
+		if (!partial) return;
+
+		await rm(partial.path, { force: true, recursive: true });
+		await this.domain.server.prisma.partialFile.delete({ where: { id, path: partial.path } });
+
+		clearTimeout(partial.timeout);
+		this.partials.delete(id);
+	}
+}

apps/server/src/lib/Config.ts

@@ -30,19 +30,18 @@ export default class Config {
 	public constructor(public server: Server) {}
 
 	public async start() {
-		const dataDir = join(process.cwd(), "..", "..", "data");
-		await mkdir(join(dataDir, "logs"), { recursive: true });
-		await mkdir(join(dataDir, "files"), { recursive: true });
-		await mkdir(join(dataDir, "paste-bins"), { recursive: true });
-		await mkdir(join(dataDir, "backups", "archives"), { recursive: true });
-		await mkdir(join(dataDir, "backups", "temp"), { recursive: true });
+		await mkdir(join(Config.dataDirectory, "logs"), { recursive: true });
+		await mkdir(join(Config.dataDirectory, "files"), { recursive: true });
+		await mkdir(join(Config.dataDirectory, "paste-bins"), { recursive: true });
+		await mkdir(join(Config.dataDirectory, "backups", "archives"), { recursive: true });
+		await mkdir(join(Config.dataDirectory, "backups", "temp"), { recursive: true });
 
 		// Check if mime-types/extensions are already updated
-		const flagsRaw = await readFile(join(dataDir, "flags.json"), "utf-8").catch(() => "{}");
+		const flagsRaw = await readFile(join(Config.dataDirectory, "flags.json"), "utf-8").catch(() => "{}");
 		const flags: Record<string, boolean> = JSON.parse(flagsRaw);
 
 		await this.updateFiles(flags["mime-type"], flags.extension);
-		await writeFile(join(dataDir, "flags.json"), JSON.stringify({ "mime-type": true, extension: true }));
+		await writeFile(join(Config.dataDirectory, "flags.json"), JSON.stringify({ "mime-type": true, extension: true }));
 
 		const config = Config.getEnv();
 		await Config.updateEnv(config);
@@ -119,6 +118,7 @@ export default class Config {
 	}
 
 	public static logger = new Logger({ name: "CONFIG" });
+	public static dataDirectory = join(process.cwd(), "..", "..", "data");
 
 	public static get VERSION(): string {
 		const file = readFileSync(join(process.cwd(), "..", "..", "package.json"), "utf-8");