Add computed fields to SQL (#92)

* Add computed fields to SQL

* Refactor code

* fix tests

* fix conflicts

src/lib/databaseUtils.ts

@@ -4,7 +4,7 @@ import path from "path";
 import { WikiLink } from "./parseFile.js";
 
 export async function resetDatabaseTables(db: Knex) {
-  const tableNames = [MddbFile, MddbTag, MddbFileTag, MddbLink];
+  const tableNames = [MddbTag, MddbFileTag, MddbLink];
   // Drop and Create tables
   for (const table of tableNames) {
     await table.deleteTable(db);
@@ -13,8 +13,8 @@ export async function resetDatabaseTables(db: Knex) {
 }
 
 export function mapFileToInsert(file: any) {
-  const { _id, file_path, extension, url_path, filetype, metadata } = file;
-  return { _id, file_path, extension, url_path, filetype, metadata };
+  const { tags, links, ...rest } = file;
+  return { ...rest };
 }
 
 export function mapLinksToInsert(filesToInsert: File[], file: any) {
@@ -67,3 +67,17 @@ export function getUniqueValues<T>(inputArray: T[]): T[] {
 
   return uniqueArray;
 }
+
+export function getUniqueProperties(objects: any[]): string[] {
+  const uniqueProperties: string[] = [];
+
+  for (const object of objects) {
+    for (const key of Object.keys(object)) {
+      if (!uniqueProperties.includes(key)) {
+        uniqueProperties.push(key);
+      }
+    }
+  }
+
+  return uniqueProperties;
+}

src/lib/indexFolder.ts

@@ -40,7 +40,7 @@ export function indexFolder(
       if (!result.success) {
         const error: ZodError = (result as any).error;
 
-        error.errors.forEach((err) => {
+        error.errors.forEach((err: any) => {
           const errorMessage = `Error: In ${
             fileObject.file_path
           } for the ${documentType} schema. \n    In "${err.path.join(

src/lib/markdowndb.ts

@@ -10,6 +10,7 @@ import {
   isLinkToDefined,
   mapFileTagsToInsert,
   getUniqueValues,
+  getUniqueProperties,
 } from "./databaseUtils.js";
 import fs from "fs";
 import { CustomConfig } from "./CustomConfig.js";
@@ -85,8 +86,6 @@ export class MarkdownDB {
     customConfig?: CustomConfig;
     watch?: boolean;
   }) {
-    await resetDatabaseTables(this.db);
-
     const fileObjects = indexFolder(
       folderPath,
       pathToUrlResolver,
@@ -152,6 +151,9 @@ export class MarkdownDB {
 
   private async saveDataToDisk(fileObjects: FileInfo[]) {
     await resetDatabaseTables(this.db);
+    const properties = getUniqueProperties(fileObjects);
+    MddbFile.deleteTable(this.db);
+    await MddbFile.createTable(this.db, properties);
 
     const filesToInsert = fileObjects.map(mapFileToInsert);
     const uniqueTags = getUniqueValues(

src/lib/schema.ts

@@ -25,11 +25,20 @@ interface File {
   url_path: string | null;
   filetype: string | null;
   metadata: MetaData | null;
+  [key: string]: any;
 }
 
 class MddbFile {
   static table = Table.Files;
   static supportedExtensions = ["md", "mdx"];
+  static defaultProperties = [
+    "_id",
+    "file_path",
+    "extension",
+    "url_path",
+    "filetype",
+    "metadata",
+  ];
 
   _id: string;
   file_path: string;
@@ -39,36 +48,42 @@ class MddbFile {
   // and another one for many-to-many relationship between files and filetypes
   filetype: string | null;
   metadata: MetaData | null;
+  [key: string]: any;
 
   // TODO type?
   constructor(file: any) {
-    this._id = file._id;
-    this.file_path = file.file_path;
-    this.extension = file.extension;
-    this.url_path = file.url_path;
-    this.filetype = file.filetype;
-    this.metadata = file.metadata ? JSON.parse(file.metadata) : null;
+    if (!file) {
+      return;
+    }
+    Object.keys(file).forEach((key) => {
+      if (key === "metadata") {
+        this[key] = file[key] ? JSON.parse(file[key]) : null;
+      } else {
+        this[key] = file[key];
+      }
+    });
   }
 
   toObject(): File {
-    return {
-      _id: this._id,
-      file_path: this.file_path,
-      extension: this.extension,
-      url_path: this.url_path,
-      filetype: this.filetype,
-      metadata: this.metadata,
-    };
+    return { ...this.file };
   }
 
-  static async createTable(db: Knex) {
+  static async createTable(db: Knex, properties: string[]) {
     const creator = (table: Knex.TableBuilder) => {
       table.string("_id").primary();
-      table.string("file_path").unique().notNullable(); // Path relative to process.cwd()
-      table.string("extension").notNullable(); // File extension
-      table.string("url_path"); // Sluggfied path relative to content folder
-      table.string("filetype"); // Type field in frontmatter if it exists
-      table.string("metadata"); // All frontmatter data
+      table.string("file_path").unique().notNullable();
+      table.string("extension").notNullable();
+      table.string("url_path");
+      table.string("filetype");
+      table.string("metadata");
+      properties.forEach((property) => {
+        if (
+          MddbFile.defaultProperties.indexOf(property) === -1 &&
+          ["tags", "links"].indexOf(property) === -1
+        ) {
+          table.string(property);
+        }
+      });
     };
     const tableExists = await db.schema.hasTable(this.table);
 
@@ -88,11 +103,24 @@ class MddbFile {
     if (!areUniqueObjectsByKey(files, "file_path")) {
       throw new Error("Files must have unique file_path");
     }
+
     const serializedFiles = files.map((file) => {
-      return {
-        ...file,
-        metadata: JSON.stringify(file.metadata),
-      };
+      const serializedFile: any = {};
+
+      Object.keys(file).forEach((key) => {
+        const value = file[key];
+        // If the value is undefined, default it to null
+        if (value !== undefined) {
+          const shouldStringify =
+            key === "metadata" || !MddbFile.defaultProperties.includes(key);
+          // Stringify all user-defined fields and metadata
+          serializedFile[key] = shouldStringify ? JSON.stringify(value) : value;
+        } else {
+          serializedFile[key] = null;
+        }
+      });
+
+      return serializedFile;
     });
 
     return db.batchInsert(Table.Files, serializedFiles);