add storage cleaner

Introduce the ability to clean up old messages from the sqlite db.
The StoragePolicy can be chosen by the user. Currently there's
two versions, delete everything based on age is the obvious.

The other is for the data hoarders among us. It'll only delete
message types which can be considered low value... Types with
a time aspect like away / back... joins / parts etc.

It tries to do that in a sensible way, so that we don't block
all other db writers that are ongoing.
The "periodically" interval is by design not exposed to the user.

client/tsconfig.json

@@ -11,6 +11,7 @@
     "../server/log.ts",
     "../server/config.ts",
     "../server/client.ts",
+    "../server/storageCleaner.ts",
     "../server/clientManager.ts",
     "../server/identification.ts",
     "../server/plugins/changelog.ts",

defaults/config.js

@@ -304,6 +304,26 @@ module.exports = {
 	// This value is set to `["sqlite", "text"]` by default.
 	messageStorage: ["sqlite", "text"],
 
+	// ### `storagePolicy`
+
+	// When the sqlite storage is in use, control the maximum storage duration.
+	// A background task will periodically clean up messages older than the limit.
+
+	// The available keys for the `storagePolicy` object are:
+	//
+	// - `enabled`: If this is false, the cleaning task is not running.
+	// - `maxAgeDays`: Maximum age of an entry in days.
+	// - `deletionPolicy`: Controls what types of messages are being deleted.
+	//   Valid options are:
+	//   - `statusOnly`: Only delete message types which are status related (e.g. away, back, join, parts, mode, ctcp...)
+	//     but keep actual messages from nicks. This keeps the DB size down while retaining "precious" messages.
+	//   - `everything`: Delete everything, including messages from irc nicks
+	storagePolicy: {
+		enabled: false,
+		maxAgeDays: 7,
+		deletionPolicy: "statusOnly",
+	},
+
 	// ### `useHexIp`
 	//
 	// When set to `true`, users' IP addresses will be encoded as hex.

server/client.ts

@@ -18,6 +18,7 @@ import TextFileMessageStorage from "./plugins/messageStorage/text";
 import Network, {IgnoreListItem, NetworkConfig, NetworkWithIrcFramework} from "./models/network";
 import ClientManager from "./clientManager";
 import {MessageStorage, SearchQuery, SearchResponse} from "./plugins/messageStorage/types";
+import {StorageCleaner} from "./storageCleaner";
 
 type OrderItem = Chan["id"] | Network["uuid"];
 type Order = OrderItem[];
@@ -138,6 +139,15 @@ class Client {
 		if (!Config.values.public && client.config.log) {
 			if (Config.values.messageStorage.includes("sqlite")) {
 				client.messageProvider = new SqliteMessageStorage(client.name);
+
+				if (Config.values.storagePolicy.enabled) {
+					log.info(
+						`Activating storage cleaner. Policy: ${Config.values.storagePolicy.deletionPolicy}. MaxAge: ${Config.values.storagePolicy.maxAgeDays} days`
+					);
+					const cleaner = new StorageCleaner(client.messageProvider);
+					cleaner.start();
+				}
+
 				client.messageStorage.push(client.messageProvider);
 			}
 

server/command-line/storage.ts

@@ -3,14 +3,15 @@ import {Command} from "commander";
 import ClientManager from "../clientManager";
 import Utils from "./utils";
 import SqliteMessageStorage from "../plugins/messageStorage/sqlite";
+import {StorageCleaner} from "../storageCleaner";
 
 const program = new Command("storage").description(
 	"various utilities related to the message storage"
 );
 
 program
 	.command("migrate")
-	.argument("[user]", "migrate a specific user only, all if not provided")
+	.argument("[username]", "migrate a specific user only, all if not provided")
 	.description("Migrate message storage where needed")
 	.on("--help", Utils.extraHelp)
 	.action(function (user) {
@@ -20,7 +21,19 @@ program
 		});
 	});
 
-async function runMigrations(user: string) {
+program
+	.command("clean")
+	.argument("[user]", "clean messages for a specific user only, all if not provided")
+	.description("Delete messages from the DB based on the storage policy")
+	.on("--help", Utils.extraHelp)
+	.action(function (user) {
+		runCleaning(user).catch((err) => {
+			log.error(err.toString());
+			process.exit(1);
+		});
+	});
+
+async function runMigrations(user?: string) {
 	const manager = new ClientManager();
 	const users = manager.getUsers();
 
@@ -65,4 +78,46 @@ function isUserLogEnabled(manager: ClientManager, user: string): boolean {
 	return conf.log;
 }
 
+async function runCleaning(user: string) {
+	const manager = new ClientManager();
+	const users = manager.getUsers();
+
+	if (user) {
+		if (!users.includes(user)) {
+			throw new Error(`invalid user ${user}`);
+		}
+
+		return cleanUser(manager, user);
+	}
+
+	for (const name of users) {
+		await cleanUser(manager, name);
+		// if any migration fails we blow up,
+		// chances are the rest won't complete either
+	}
+}
+
+async function cleanUser(manager: ClientManager, user: string) {
+	log.info("handling user", user);
+
+	if (!isUserLogEnabled(manager, user)) {
+		log.info("logging disabled for user", user, ". Skipping");
+		return;
+	}
+
+	const sqlite = new SqliteMessageStorage(user);
+	await sqlite.enable();
+	const cleaner = new StorageCleaner(sqlite);
+	const num_deleted = await cleaner.runDeletesNoLimit();
+	log.info(`deleted ${num_deleted} messages`);
+	log.info("running a vacuum now, this might take a while");
+
+	if (num_deleted > 0) {
+		await sqlite.vacuum();
+	}
+
+	await sqlite.close();
+	log.info(`cleaning messages for ${user} has been successful`);
+}
+
 export default program;

server/config.ts

@@ -76,6 +76,12 @@ type Debug = {
 	raw: boolean;
 };
 
+type StoragePolicy = {
+	enabled: boolean;
+	maxAgeDays: number;
+	deletionPolicy: "statusOnly" | "everything";
+};
+
 export type ConfigType = {
 	public: boolean;
 	host: string | undefined;
@@ -97,6 +103,7 @@ export type ConfigType = {
 	defaults: Defaults;
 	lockNetwork: boolean;
 	messageStorage: string[];
+	storagePolicy: StoragePolicy;
 	useHexIp: boolean;
 	webirc?: WebIRC;
 	identd: Identd;