feat: extensible table

CHANGELOG.org

@@ -14,6 +14,7 @@
 
 - Include attachment directory in =vulpea-note= and =notes= table.
 - [[https://github.com/d12frosted/vulpea/issues/130][vulpea#130]] Introduce mechanism to automatically rebuild database when needed. This happens either on the first usage of Vulpea or when =vulpea-db-version= increases.
+- [[https://github.com/d12frosted/vulpea/pull/158][vulpea#158]] Introduce mechanism to define more tables in Org Roam database (see =vulpea-db-define-table=). And provide a hook to fill defined tables with data (see =vulpea-db-insert-note-functions=).
 
 ** v0.3.0
 :PROPERTIES:

README.org

@@ -1041,6 +1041,33 @@ As you can see, =vulpea-db-query= doesn't allow to pass any custom SQL for filte
 - =vulpea-db-get-file-by-id= - function to get =FILE= of a note represented by =ID=. Supports headings of the note.
 - =vulpea-db-search-by-title= - function to query notes with =TITLE=.
 
+**** Extending database
+:PROPERTIES:
+:ID:                     8d0b72a8-b5ee-4be8-a17f-84b151ad85fc
+:END:
+
+You may extend database by adding custom tables using =vulpea-db-define-table=:
+
+#+begin_src emacs-lisp
+  (vulpea-db-define-table
+   ;; name
+   'my-custom-table
+   ;; version
+   1
+   ;; schema
+   '([(note-id :unique :primary-key)
+      (some-column :not-null)
+      (some-other-column)]
+     ;; useful to automatically cleanup your table whenever a note/node/file is removed
+     (:foreign-key [note-id] :references nodes [id] :on-delete :cascade))
+   ;; index
+   '((custom-node-id-index [note-id])))
+#+end_src
+
+Consult with [[https://github.com/magit/emacsql/][magit/emacsql]] documentation to learn more about schema and indices.
+
+In order to populate your table with data, you should add a hook to =vulpea-db-insert-note-functions=. It is called with a single argument of type =vulpea-note= (keep in mind that =vulpea-note-links= slot is empty, open a ticket if you need it).
+
 *** =vulpea-meta=
 :PROPERTIES:
 :ID:                     9bb0311f-c257-46f1-8e1f-68c735a1a07c

test/vulpea-db-test.el

@@ -898,47 +898,50 @@
     (org-roam-db-sync 'force)
 
     ;; initially there are no vulpea specific tables
-    (pcase-dolist (`(,table ,_) vulpea-db--schemata)
-      (expect (org-roam-db-query
-               [:select name
-                :from sqlite_master
-                :where (and (= type 'table)
-                            (= name $r1))]
-               (emacsql-escape-identifier table))
-              :to-equal nil))
-    (pcase-dolist (`(,index-name ,_ ,_) vulpea-db--indices)
-      (expect (org-roam-db-query
-               [:select name
-                :from sqlite_master
-                :where (and (= type 'index)
-                            (= name $r1))]
-               (emacsql-escape-identifier index-name))
-              :to-equal nil))
+    (-each vulpea-db--tables
+      (-lambda ((table _ _ indices))
+        (expect (org-roam-db-query
+                 [:select name
+                  :from sqlite_master
+                  :where (and (= type 'table)
+                              (= name $r1))]
+                 (emacsql-escape-identifier table))
+                :to-equal nil)
+        (-each indices
+          (-lambda ((index-name))
+            (expect (org-roam-db-query
+                     [:select name
+                      :from sqlite_master
+                      :where (and (= type 'index)
+                                  (= name $r1))]
+                     (emacsql-escape-identifier index-name))
+                    :to-equal nil)))))
 
     ;; then we setup vulpea-db
     (message "vulpea-db-setup")
     (vulpea-db-autosync-enable)
 
     ;; and vulpea specific tables should exist
-    (pcase-dolist (`(,table ,_) vulpea-db--schemata)
-      (expect (org-roam-db-query
-               [:select name
-                :from sqlite_master
-                :where (and (= type 'table)
-                            (= name $r1))]
-               (emacsql-escape-identifier table))
-              :to-equal (list (list (intern (emacsql-escape-identifier table))))))
-    (pcase-dolist (`(,index-name ,_ ,_) vulpea-db--indices)
-      (expect (org-roam-db-query
-               [:select name
-                :from sqlite_master
-                :where (and (= type 'index)
-                            (= name $r1))]
-               (emacsql-escape-identifier index-name))
-              :to-equal (list (list (intern (emacsql-escape-identifier index-name))))))
-    (expect (caar (org-roam-db-query [:select version :from cache :where (= id "vulpea")]))
-            :to-equal
-            vulpea-db-version)
+    (-each vulpea-db--tables
+      (-lambda ((table version _ indices))
+        (expect (org-roam-db-query
+                 [:select name
+                  :from sqlite_master
+                  :where (and (= type 'table)
+                              (= name $r1))]
+                 (emacsql-escape-identifier table))
+                :to-equal (list (list (intern (emacsql-escape-identifier table)))))
+        (-each indices
+          (-lambda ((index-name))
+            (expect (org-roam-db-query
+                     [:select name
+                      :from sqlite_master
+                      :where (and (= type 'index)
+                                  (= name $r1))]
+                     (emacsql-escape-identifier index-name))
+                    :to-equal (list (list (intern (emacsql-escape-identifier index-name)))))))
+        (expect (caar (org-roam-db-query [:select version :from versions :where (= id $s1)] table))
+                :to-equal version)))
 
     ;; sync a file
     (message "update file")
@@ -975,5 +978,72 @@
                      ("answer" . ("42")))
              :attach-dir (expand-file-name "data/05/907606-f836-45bf-bd36-a8444308eddd" org-roam-directory)))))
 
+(describe "vulpea-db-insert-note-functions"
+  (before-each
+    (vulpea-test--init)
+    (spy-on 'insert-handle-fn)
+    (add-hook 'vulpea-db-insert-note-functions #'insert-handle-fn))
+
+  (after-each
+    (vulpea-test--teardown)
+    (remove-hook 'vulpea-db-insert-note-functions #'insert-handle-fn))
+
+  (it "calls insert hook on file level note"
+    (let* ((id "1cc15044-aedb-442e-b727-9e3f7346be95")
+           (note (vulpea-db-get-by-id id))
+           (file (vulpea-note-path note)))
+      ;; force sync of a single file by clearing it and syncing manually
+      (org-roam-db-clear-file file)
+      (org-roam-db-sync)
+      (expect 'insert-handle-fn :to-have-been-called-times 1)
+      (expect 'insert-handle-fn :to-have-been-called-with
+              (make-vulpea-note
+               :path (expand-file-name "note-with-link.org" org-roam-directory)
+               :title "Note with link"
+               :tags nil
+               :level 0
+               :id "1cc15044-aedb-442e-b727-9e3f7346be95"
+               :links nil               ; not supported in this hook
+               :properties (list
+                            (cons "CATEGORY" "note-with-link")
+                            (cons "ID" "1cc15044-aedb-442e-b727-9e3f7346be95")
+                            (cons "BLOCKED" "")
+                            (cons "FILE" (expand-file-name "note-with-link.org" org-roam-directory))
+                            (cons "PRIORITY" "B"))
+               :attach-dir (expand-file-name "data/1c/c15044-aedb-442e-b727-9e3f7346be95" org-roam-directory)))))
+
+  (it "calls insert hook on outline level note"
+    (let* ((id "1cc15044-aedb-442e-b727-9e3f7346be95"))
+      ;; force sync by adding a new header to the end of existing note
+      (vulpea-utils-with-note (vulpea-db-get-by-id id)
+        (goto-char (point-max))
+        (insert
+         "\n"
+         "* I was added\n")
+        (setf id (org-id-get-create))
+        (save-buffer))
+      (expect 'insert-handle-fn :to-have-been-called-times 2)
+      (expect 'insert-handle-fn :to-have-been-called-with
+              (make-vulpea-note
+               :path (expand-file-name "note-with-link.org" org-roam-directory)
+               :title "I was added"
+               :tags nil
+               :level 1
+               :id id
+               :links nil               ; not supported in this hook
+               :properties (list
+                            (cons "CATEGORY" "note-with-link")
+                            (cons "ID" id)
+                            (cons "BLOCKED" "")
+                            (cons "FILE" (expand-file-name "note-with-link.org" org-roam-directory))
+                            (cons "PRIORITY" "B")
+                            (cons "ITEM" "I was added"))
+               :attach-dir (expand-file-name
+                            (concat "data/"
+                                    (substring-no-properties id 0 2)
+                                    "/"
+                                    (substring-no-properties id 2))
+                            org-roam-directory))))))
+
 (provide 'vulpea-db-test)
 ;;; vulpea-db-test.el ends here

test/vulpea-perf-test.el

@@ -38,7 +38,7 @@
 (require 'org-roam)
 (require 'vulpea)
 
-(defconst vulpea-perf-zip-branch "attach-dir")
+(defconst vulpea-perf-zip-branch "extensible-table")
 
 (defconst vulpea-perf-zip-url
   (format