core: fix database disk image malformed error (#5525)

packages/core/__tests__/fts-triggers.test.ts

@@ -18,7 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { expect, test } from "vitest";
-import { TEST_NOTE, databaseTest, noteTest } from "./utils";
+import { databaseTest, noteTest } from "./utils";
 
 test("updating deleted content should not throw", () =>
   databaseTest().then(async (db) => {

packages/core/src/api/lookup.ts

@@ -25,6 +25,7 @@ import { AnyColumnWithTable, Kysely, sql } from "kysely";
 import { FilteredSelector } from "../database/sql-collection";
 import { VirtualizedGrouping } from "../utils/virtualized-grouping";
 import { logger } from "../logger";
+import { rebuildSearchIndex } from "../database/fts";
 
 type SearchResults<T> = {
   sorted: (limit?: number) => Promise<VirtualizedGrouping<T>>;
@@ -245,4 +246,9 @@ export default class Lookup {
     if (!ids.length) return [];
     return selector.items(ids);
   }
+
+  async rebuild() {
+    const db = this.db.sql() as unknown as Kysely<RawDatabaseSchema>;
+    await rebuildSearchIndex(db);
+  }
 }

packages/core/src/database/fts.ts

@@ -0,0 +1,74 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Kysely, sql } from "kysely";
+import { RawDatabaseSchema } from ".";
+
+export async function rebuildSearchIndex(db: Kysely<RawDatabaseSchema>) {
+  await db.transaction().execute(async (tx) => {
+    for (const query of [
+      sql`INSERT INTO content_fts(content_fts) VALUES('delete-all')`,
+      sql`INSERT INTO notes_fts(notes_fts) VALUES('delete-all')`
+    ]) {
+      await query.execute(tx);
+    }
+
+    await tx
+      .insertInto("content_fts")
+      .columns(["rowid", "id", "data", "noteId"])
+      .expression((eb) =>
+        eb
+          .selectFrom("content")
+          .where((eb) =>
+            eb.and([
+              eb("noteId", "is not", null),
+              eb("data", "is not", null),
+              eb("deleted", "is not", true)
+            ])
+          )
+          .select([
+            "rowid",
+            "id",
+            sql`IIF(locked == 1, '', data)`.as("data"),
+            "noteId"
+          ])
+      )
+      .execute();
+
+    await tx
+      .insertInto("notes_fts")
+      .columns(["rowid", "id", "title"])
+      .expression((eb) =>
+        eb
+          .selectFrom("notes")
+          .where((eb) =>
+            eb.and([eb("title", "is not", null), eb("deleted", "is not", true)])
+          )
+          .select(["rowid", "id", "title"])
+      )
+      .execute();
+
+    for (const query of [
+      sql`INSERT INTO content_fts(content_fts) VALUES('optimize')`,
+      sql`INSERT INTO notes_fts(notes_fts) VALUES('optimize')`
+    ]) {
+      await query.execute(tx);
+    }
+  });
+}

packages/core/src/database/migrations.ts

@@ -24,6 +24,7 @@ import {
   MigrationProvider,
   sql
 } from "kysely";
+import { rebuildSearchIndex } from "./fts";
 
 const COLLATE_NOCASE: ColumnBuilderCallback = (col) =>
   col.modifyEnd(sql`collate nocase`);
@@ -283,6 +284,11 @@ export class NNMigrationProvider implements MigrationProvider {
             .execute();
         },
         async down(db) {}
+      },
+      "2": {
+        async up(db) {
+          await rebuildSearchIndex(db);
+        }
       }
     };
   }

packages/core/src/database/triggers.ts

@@ -31,16 +31,16 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {
     .addEvent("insert")
     .when((eb) =>
       eb.and([
-        eb("new.deleted", "is not", true),
-        eb("new.locked", "is not", true),
-        eb("new.data", "is not", null)
+        eb("new.noteId", "is not", null),
+        eb("new.data", "is not", null),
+        eb("new.deleted", "is not", true)
       ])
     )
     .addQuery((c) =>
       c.insertInto("content_fts").values({
         rowid: sql`new.rowid`,
         id: sql`new.id`,
-        data: sql`new.data`,
+        data: sql`IIF(new.locked == 1, '', new.data)`,
         noteId: sql`new.noteId`
       })
     )
@@ -53,6 +53,13 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {
     .onTable("content", "main")
     .after()
     .addEvent("delete")
+    .when((eb) =>
+      eb.and([
+        eb("old.noteId", "is not", null),
+        eb("old.data", "is not", null),
+        eb("old.deleted", "is not", true)
+      ])
+    )
     .addQuery((c) =>
       c.insertInto("content_fts").values({
         content_fts: sql.lit("delete"),
@@ -73,9 +80,9 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {
     .addEvent("update")
     .when((eb) =>
       eb.and([
-        eb("old.deleted", "is not", true),
         eb("old.noteId", "is not", null),
-        eb("old.data", "is not", null)
+        eb("old.data", "is not", null),
+        eb("old.deleted", "is not", true)
       ])
     )
     .addQuery((c) =>
@@ -107,7 +114,8 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {
     .addEvent("insert")
     .when((eb) =>
       eb.and([
-        eb.or([eb("new.deleted", "is", null), eb("new.deleted", "==", false)])
+        eb("new.title", "is not", null),
+        eb("new.deleted", "is not", true)
       ])
     )
     .addQuery((c) =>
@@ -126,6 +134,12 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {
     .onTable("notes", "main")
     .after()
     .addEvent("delete")
+    .when((eb) =>
+      eb.and([
+        eb("old.title", "is not", null),
+        eb("old.deleted", "is not", true)
+      ])
+    )
     .addQuery((c) =>
       c.insertInto("notes_fts").values({
         notes_fts: sql.lit("delete"),