Thumbnail management

AGENTS.md

@@ -54,6 +54,10 @@ hollo/
 │   │   ├── v1/                   # API v1 endpoints
 │   │   └── v2/                   # API v2 endpoints (search, notifications)
 │   │
+│   ├── cleanup/                  # Cleanup (of old data)
+│   │   ├── processors.ts         # Cleanup item processors
+│   │   └── worker.ts             # Background job worker
+│   │
 │   ├── components/               # Hono JSX components (server-rendered)
 │   │
 │   ├── entities/                 # Entity serialization (DB → API response)
@@ -111,6 +115,7 @@ Key architectural components
     responses
  -  *Pages* (*src/pages/*): Web UI pages (profile, setup, dashboard, etc.)
  -  *Import system* (*src/import/*): Background job processing for data imports
+ -  *Cleanup system* (*src/cleanup/*): Background job processing for cleanup actions
 
 ### Key files
 

CHANGES.md

@@ -120,6 +120,13 @@ To be released.
     editor and is now returned from `GET /api/v1/preferences`, which helps
     clients like Phanpy honor each account's preferred CW behavior.  [[#425]]
 
+ -  Added a new dashboard page for thumbnail cleanup at `/thumbnail_cleanup`.
+    Thumbnails from remote posts that have not been bookmarked, liked, reacted
+    to, shared nor quoted by a local account before a given cut-off data can
+    be mass-deleted in order to lower storage demand. The original posts are
+    not deleted, neither is the relationship to the original media nor the ALT
+    text.
+
  -  Upgraded Fedify to 2.1.7.
 
 [#348]: https://github.com/fedify-dev/hollo/issues/348

bin/server.ts

@@ -65,14 +65,21 @@ if (NODE_TYPE === "web" || NODE_TYPE === "all") {
 }
 
 // Start workers if running as worker or all node
-let stopWorker: (() => void) | undefined;
+let stopWorkers: (() => void) | undefined;
 if (NODE_TYPE === "worker" || NODE_TYPE === "all") {
-  const [{ federation }, { startImportWorker, stopImportWorker }] =
-    await Promise.all([
-      import("../src/federation"),
-      import("../src/import/worker"),
-    ]);
-  stopWorker = stopImportWorker;
+  const [
+    { federation },
+    { startImportWorker, stopImportWorker },
+    { startCleanupWorker, stopCleanupWorker },
+  ] = await Promise.all([
+    import("../src/federation"),
+    import("../src/import/worker"),
+    import("../src/cleanup/worker"),
+  ]);
+  stopWorkers = () => {
+    stopImportWorker();
+    stopCleanupWorker();
+  };
 
   // Start the Fedify message queue
   const controller = new AbortController();
@@ -83,17 +90,18 @@ if (NODE_TYPE === "worker" || NODE_TYPE === "all") {
       process.exit(1);
     });
 
-  // Start the import worker for background job processing
+  // Start the workers for background job processing
   startImportWorker();
+  startCleanupWorker();
 
-  console.log("Worker started (Fedify queue + Import worker)");
+  console.log("Worker started (Fedify queue + Import worker + Cleanup worker)");
 }
 
 // Graceful shutdown handling
 const shutdown = () => {
   if (NODE_TYPE === "worker" || NODE_TYPE === "all") {
     console.log("Stopping workers...");
-    stopWorker?.();
+    stopWorkers?.();
   }
   process.exit(0);
 };

docs/src/content/docs/install/env.mdx

@@ -46,9 +46,9 @@ e.g., `UTC`, `America/New_York`, `Asia/Tokyo`.
 
 Controls which components run in this process.  Valid values are:
 
- -  `all` (default): Run web server, Fedify message queue, and import worker
+ -  `all` (default): Run web server, Fedify message queue, import worker and cleanup worker
  -  `web`: Run only the web server (HTTP API)
- -  `worker`: Run only workers (Fedify message queue + import worker)
+ -  `worker`: Run only workers (Fedify message queue + import worker + cleanup worker)
 
 This allows separating the web server from background workers for better
 scalability.  When running high-traffic instances with many followers,

docs/src/content/docs/install/workers.mdx

@@ -35,15 +35,16 @@ Hollo has three main components:
  1. *Web server*: Handles HTTP requests (API, web UI)
  2. *Fedify message queue*: Processes ActivityPub inbox/outbox messages
  3. *Import worker*: Handles background data import jobs
+ 4. *Cleanup worker*: Handles background cleanup jobs
 
 By default (`NODE_TYPE=all`), all three run in a single process. You can
 separate them using the `NODE_TYPE` environment variable:
 
-| `NODE_TYPE` | Web server | Fedify queue | Import worker |
-| ----------- | ---------- | ------------ | ------------- |
-| `all` (default) | ✓ | ✓ | ✓ |
-| `web` | ✓ | ✗ | ✗ |
-| `worker` | ✗ | ✓ | ✓ |
+| `NODE_TYPE` | Web server | Fedify queue | Import worker | Cleanup worker |
+| ----------- | ---------- | ------------ | ------------- | -------------- |
+| `all` (default) | ✓ | ✓ | ✓ | ✓ |
+| `web` | ✓ | ✗ | ✗ | ✗ |
+| `worker` | ✗ | ✓ | ✓ | ✓ |
 
 All nodes share the same PostgreSQL database, which acts as the message queue
 backend using `LISTEN`/`NOTIFY` for real-time message delivery.
@@ -159,7 +160,8 @@ NODE_TYPE=worker pnpm prod
 pnpm worker
 ```
 
-This starts the Fedify message queue and import worker without the web server.
+This starts the Fedify message queue, import worker and cleanup worker without
+the web server.
 
 <Aside type="caution">
   When running `NODE_TYPE=web`, ensure at least one `NODE_TYPE=worker` node
@@ -251,11 +253,11 @@ sudo journalctl -u hollo-worker -f
 When a worker node starts, you should see:
 
 ```
-Worker started (Fedify queue + Import worker)
+Worker started (Fedify queue + Import worker + Cleanup worker)
 ```
 
-Watch for messages about processing activities and import jobs to confirm
-the worker is functioning correctly.
+Watch for messages about processing activities and import and cleanup jobs to 
+confirm the worker is functioning correctly.
 
 
 Troubleshooting

drizzle/0080_add_thumbnail_cleaned_flag.sql

@@ -0,0 +1 @@
+ALTER TABLE "media" ADD COLUMN "thumbnail_cleaned" boolean DEFAULT false NOT NULL;

drizzle/0081_add_cleanup_jobs.sql

@@ -0,0 +1,29 @@
+CREATE TYPE "public"."cleanup_job_category" AS ENUM('cleanup_thumbnails');--> statement-breakpoint
+CREATE TYPE "public"."cleanup_job_status" AS ENUM('pending', 'processing', 'completed', 'failed', 'cancelled');--> statement-breakpoint
+CREATE TABLE "cleanup_job_items" (
+	"id" uuid PRIMARY KEY NOT NULL,
+	"job_id" uuid NOT NULL,
+	"status" "cleanup_job_status" DEFAULT 'pending' NOT NULL,
+	"data" jsonb NOT NULL,
+	"error_message" text,
+	"processed_at" timestamp with time zone,
+	"created" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "cleanup_jobs" (
+	"id" uuid PRIMARY KEY NOT NULL,
+	"category" "cleanup_job_category" NOT NULL,
+	"status" "cleanup_job_status" DEFAULT 'pending' NOT NULL,
+	"total_items" integer DEFAULT 0 NOT NULL,
+	"processed_items" integer DEFAULT 0 NOT NULL,
+	"successful_items" integer DEFAULT 0 NOT NULL,
+	"failed_items" integer DEFAULT 0 NOT NULL,
+	"error_message" text,
+	"created" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	"started_at" timestamp with time zone,
+	"completed_at" timestamp with time zone
+);
+--> statement-breakpoint
+ALTER TABLE "cleanup_job_items" ADD CONSTRAINT "cleanup_job_items_job_id_cleanup_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."cleanup_jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "cleanup_job_items_job_id_status_index" ON "cleanup_job_items" USING btree ("job_id","status");--> statement-breakpoint
+CREATE INDEX "cleanup_jobs_status_created_index" ON "cleanup_jobs" USING btree ("status","created");