enhance: introduce sqlite as a plugabble metadata store

.github/workflows/ci.yml

@@ -79,7 +79,7 @@ jobs:
         with:
           path: |
             packages/*/lib/*
-          key: ${{ runner.os }}-${{ hashFiles('yarn.lock') }}-21
+          key: ${{ runner.os }}-${{ hashFiles('yarn.lock') }}-22
           restore-keys: |
             ${{ runner.os }}-yarn-9
 

.gitignore

@@ -33,3 +33,6 @@ packages/nextjs-template/public
 .npmrc
 .env.production
 packages/dendron-viz/lib
+generated-prisma-client
+packages/engine-server/prisma/dev.db
+dev.db-journal

Makefile

@@ -16,3 +16,20 @@ cleanBuild:
 	echo "Clean building..."
 	make clean
 	make install
+
+db-gen:
+	cd packages/engine-server && yarn prisma generate 
+	cd packages/engine-server && rm -rf lib/generated-prisma-client
+	cd packages/engine-server && cp -R src/generated-prisma-client lib/generated-prisma-client
+
+start-local-registry:
+	yarn config set registry http://localhost:4873
+	npm set registry http://localhost:4873/
+	npx verdaccio -c ./bootstrap/data/verdaccio/config.yaml 
+
+publish-local:
+	lerna publish from-package --ignore-scripts
+
+build-plugin:
+	dendron dev prep_plugin && rm package.json
+	dendron dev package_plugin

bootstrap/scripts/buildAll.js

@@ -13,15 +13,15 @@ const $ = (cmd) => {
 
 console.log("building all...");
 $(`npx lerna run build --scope @dendronhq/common-all`);
-$(`npx lerna run build --scope @dendronhq/common-server `);
-$(`npx lerna run build --scope @dendronhq/dendron-viz `);
-$(`npx lerna run build --scope @dendronhq/engine-server `);
-$(`npx lerna run build --scope @dendronhq/pods-core `);
+$(`npx lerna run build --scope @dendronhq/common-server`);
+$(`npx lerna run build --scope @dendronhq/dendron-viz`);
+$(`npx lerna run build --scope @dendronhq/engine-server`);
+$(`npx lerna run build --scope @dendronhq/pods-core`);
 $(
-  `npx lerna run build --parallel --scope "@dendronhq/{common-test-utils,api-server,common-assets}"`
+ `npx lerna run build --parallel --scope "@dendronhq/{common-test-utils,api-server,common-assets}"`
 );
 $(
-  `npx lerna run build --parallel --scope "@dendronhq/{common-frontend,dendron-cli}"`
+ `npx lerna run build --parallel --scope "@dendronhq/{common-frontend,dendron-cli}"`
 );
 $(`npx lerna run build --scope "@dendronhq/engine-test-utils"`);
 $(`npx lerna run build --scope "@dendronhq/dendron-plugin-views"`);

packages/common-all/data/dendron-yml.validator.json

@@ -768,6 +768,13 @@
         },
         "apiEndpoint": {
           "type": "string"
+        },
+        "metadataStore": {
+          "type": "string",
+          "enum": [
+            "sqlite",
+            "json"
+          ]
         }
       },
       "required": [

packages/common-all/src/constants/configs/workspace.ts

@@ -224,4 +224,8 @@ export const WORKSPACE: DendronConfigEntryCollection<DendronWorkspaceConfig> = {
     label: "Enable smart references",
     desc: "When enabled, note references that include a header will transclude said header and all sub headers",
   },
+  metadataStore: {
+    label: "Storage engine for metadata",
+    desc: "values: sqlite|json",
+  },
 };

packages/common-all/src/types/configs/workspace/workspace.ts

@@ -39,6 +39,7 @@ export type DendronWorkspaceConfig = {
   //
   feedback?: boolean;
   apiEndpoint?: string;
+  metadataStore?: "sqlite" | "json";
 };
 
 export type DendronWorkspace = {

packages/dendron-cli/src/commands/devCLICommand.ts

@@ -450,7 +450,7 @@ export class DevCLICommand extends CLICommand<CommandOpts, CommandOutput> {
     await this.syncAssets(opts);
 
     this.print("prep repo...");
-    await BuildUtils.prepPluginPkg();
+    await BuildUtils.prepPluginPkg(opts.extensionTarget);
 
     if (!shouldPublishLocal) {
       this.print(
@@ -468,11 +468,6 @@ export class DevCLICommand extends CLICommand<CommandOpts, CommandOutput> {
     this.print("compiling plugin...");
     await BuildUtils.compilePlugin(opts);
 
-    if (opts.extensionTarget === ExtensionTarget.NIGHTLY) {
-      this.print("modifying plugin manifest for nightly target...");
-      await BuildUtils.prepPluginPkg(ExtensionTarget.NIGHTLY);
-    }
-
     this.print("package deps...");
     await BuildUtils.packagePluginDependencies(opts);
 

packages/dendron-cli/src/utils/build.ts

@@ -221,6 +221,18 @@ export class BuildUtils {
         "@dendronhq/engine-test-utils",
       ],
     });
+
+    await fs.copy(
+      path.join(
+        this.getPluginRootPath(),
+        "..",
+        "engine-server",
+        "src",
+        "drivers",
+        "generated-prisma-client"
+      ),
+      path.join(this.getPluginRootPath(), "dist", "generated-prisma-client")
+    );
   }
 
   /**

packages/engine-server/copyPrismaClient.js

@@ -0,0 +1,12 @@
+const fs = require("fs-extra");
+const path = require("path");
+
+function main() {
+	const libPath = path.join(__dirname, "lib", "drivers", "generated-prisma-client");
+	const srcPath = path.join("src", "drivers", "generated-prisma-client");
+	fs.ensureDirSync(libPath);
+	fs.copy(srcPath, libPath, { overwrite: true });
+	console.log("done");
+}
+
+main();

packages/engine-server/package.json

@@ -23,13 +23,14 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "clean": "rimraf lib && rimraf coverage",
+    "clean": "rimraf lib && rimraf coverage && rimraf src/drivers/generated-prisma-client",
     "format": "echo nop",
     "lint": "echo stub",
+    "buildPrismaClient": "yarn prisma generate && node copyPrismaClient.js",
     "prepublishOnly": "yarn build",
     "prebuild": "yarn clean && yarn format && yarn lint && echo Using TypeScript && tsc --version",
-    "build": "yarn compile",
-    "buildCI": "yarn compile",
+    "build": "yarn buildPrismaClient && yarn compile",
+    "buildCI": "yarn compile && yarn buildPrismaClient",
     "compile": "tsc -p tsconfig.build.json ",
     "watch": "yarn compile --watch"
   },
@@ -39,6 +40,7 @@
     "@dendronhq/remark-mermaid": "^0.3.0",
     "@jcoreio/async-throttle": "^1.4.3",
     "@mapbox/rehype-prism": "^0.5.0",
+    "@prisma/client": "^4.1.1",
     "axios": "^0.21.1",
     "chokidar": "^3.5.2",
     "cross-path-sort": "^1.0.0",
@@ -53,6 +55,7 @@
     "mdast-builder": "^1.1.1",
     "mdast-util-compact": "^2.0.1",
     "mdast-util-inject": "^1.1.0",
+    "prisma": "^4.1.1",
     "qs": "^6.10.1",
     "rehype-autolink-headings": "^5.0.1",
     "rehype-katex": "5",
@@ -78,7 +81,7 @@
     "unified-engine": "^8.0.0",
     "unist-util-select": "^3.0.1",
     "unist-util-visit": "^2.0.3",
-    "vscode-uri": "^2.1.2"
+    "vscode-uri": "3.0.3"
   },
   "devDependencies": {
     "@types/execa": "^2.0.0",

packages/engine-server/prisma/migrations/20220819014343_add_davult/migration.sql

@@ -0,0 +1,24 @@
+-- CreateTable
+CREATE TABLE "notes" (
+    "fname" TEXT,
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "title" TEXT,
+    "vault" TEXT,
+    "updated" INTEGER,
+    "created" INTEGER,
+    "stub" BOOLEAN,
+    "anchors" json,
+    "desc" TEXT,
+    "links" json
+);
+
+-- CreateTable
+CREATE TABLE "DVault" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "name" TEXT,
+    "relativePath" TEXT NOT NULL,
+    "schemaVersion" INTEGER NOT NULL
+);
+
+-- CreateIndex
+CREATE INDEX "idx_notes_id" ON "notes"("id");

packages/engine-server/prisma/migrations/20220819153634_add_workspace/migration.sql

@@ -0,0 +1,59 @@
+/*
+  Warnings:
+
+  - You are about to drop the `notes` table. If the table is not empty, all the data it contains will be lost.
+  - The primary key for the `DVault` table will be changed. If it partially fails, the table could be left without primary key constraint.
+  - You are about to drop the column `id` on the `DVault` table. All the data in the column will be lost.
+  - You are about to drop the column `relativePath` on the `DVault` table. All the data in the column will be lost.
+  - You are about to drop the column `schemaVersion` on the `DVault` table. All the data in the column will be lost.
+  - Added the required column `fsPath` to the `DVault` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `wsRoot` to the `DVault` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- DropIndex
+DROP INDEX "idx_notes_id";
+
+-- DropTable
+PRAGMA foreign_keys=off;
+DROP TABLE "notes";
+PRAGMA foreign_keys=on;
+
+-- CreateTable
+CREATE TABLE "Note" (
+    "fname" TEXT,
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "title" TEXT,
+    "updated" INTEGER,
+    "created" INTEGER,
+    "stub" BOOLEAN,
+    "dVaultWsRoot" TEXT NOT NULL,
+    "dVaultFsPath" TEXT NOT NULL,
+    CONSTRAINT "Note_dVaultWsRoot_dVaultFsPath_fkey" FOREIGN KEY ("dVaultWsRoot", "dVaultFsPath") REFERENCES "DVault" ("wsRoot", "fsPath") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "Workspace" (
+    "wsRoot" TEXT NOT NULL PRIMARY KEY,
+    "prismaSchemaVersion" INTEGER NOT NULL
+);
+
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_DVault" (
+    "name" TEXT,
+    "fsPath" TEXT NOT NULL,
+    "wsRoot" TEXT NOT NULL,
+    CONSTRAINT "DVault_wsRoot_fkey" FOREIGN KEY ("wsRoot") REFERENCES "Workspace" ("wsRoot") ON DELETE CASCADE ON UPDATE CASCADE
+);
+INSERT INTO "new_DVault" ("name") SELECT "name" FROM "DVault";
+DROP TABLE "DVault";
+ALTER TABLE "new_DVault" RENAME TO "DVault";
+CREATE UNIQUE INDEX "DVault_wsRoot_fsPath_key" ON "DVault"("wsRoot", "fsPath");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;
+
+-- CreateIndex
+CREATE INDEX "idx_notes_id" ON "Note"("id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Workspace_wsRoot_key" ON "Workspace"("wsRoot");

packages/engine-server/prisma/migrations/20220819161942_update_db/migration.sql

@@ -0,0 +1,39 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `dVaultFsPath` on the `Note` table. All the data in the column will be lost.
+  - You are about to drop the column `dVaultWsRoot` on the `Note` table. All the data in the column will be lost.
+  - Added the required column `id` to the `DVault` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `dVaultId` to the `Note` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_DVault" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "name" TEXT,
+    "fsPath" TEXT NOT NULL,
+    "wsRoot" TEXT NOT NULL,
+    CONSTRAINT "DVault_wsRoot_fkey" FOREIGN KEY ("wsRoot") REFERENCES "Workspace" ("wsRoot") ON DELETE CASCADE ON UPDATE CASCADE
+);
+INSERT INTO "new_DVault" ("fsPath", "name", "wsRoot") SELECT "fsPath", "name", "wsRoot" FROM "DVault";
+DROP TABLE "DVault";
+ALTER TABLE "new_DVault" RENAME TO "DVault";
+CREATE UNIQUE INDEX "DVault_name_key" ON "DVault"("name");
+CREATE UNIQUE INDEX "DVault_wsRoot_fsPath_key" ON "DVault"("wsRoot", "fsPath");
+CREATE TABLE "new_Note" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "fname" TEXT,
+    "title" TEXT,
+    "updated" INTEGER,
+    "created" INTEGER,
+    "stub" BOOLEAN,
+    "dVaultId" INTEGER NOT NULL,
+    CONSTRAINT "Note_dVaultId_fkey" FOREIGN KEY ("dVaultId") REFERENCES "DVault" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_Note" ("created", "fname", "id", "stub", "title", "updated") SELECT "created", "fname", "id", "stub", "title", "updated" FROM "Note";
+DROP TABLE "Note";
+ALTER TABLE "new_Note" RENAME TO "Note";
+CREATE INDEX "idx_notes_id" ON "Note"("id");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

packages/engine-server/prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "sqlite"

packages/engine-server/prisma/schema.prisma

@@ -0,0 +1,43 @@
+generator client {
+  provider      = "prisma-client-js"
+  binaryTargets = ["native", "darwin-arm64", "darwin", "windows", "debian-openssl-1.1.x"]
+  output        = "./../src/drivers/generated-prisma-client"
+}
+
+datasource db {
+  provider = "sqlite"
+  url      = "file:./dev.db"
+}
+
+model Note {
+  id           String               @id
+  fname        String?
+  title        String?
+  vault        DVault               @relation(fields: [dVaultId], references: [id])
+  updated      Int?
+  created      Int?
+  stub         Boolean?
+  // anchors      Unsupported("json")?
+  // desc         String?
+  // links        Unsupported("json")?
+  dVaultId  Int
+
+  @@index([id], map: "idx_notes_id")
+}
+
+model DVault {
+  id        Int @id @default(autoincrement())
+  name      String?   @unique
+  fsPath    String
+  wsRoot    String
+  workspace Workspace @relation(fields: [wsRoot], references: [wsRoot], onDelete: Cascade)
+  Note      Note[]
+
+  @@unique([wsRoot, fsPath])
+}
+
+model Workspace {
+  wsRoot              String   @id @unique
+  prismaSchemaVersion Int
+  vaults              DVault[]
+}