From 6577fb1831cddd86dafc1f167bbd37eda43376d1 Mon Sep 17 00:00:00 2001 From: Boris Buliga Date: Wed, 9 Nov 2022 09:24:35 +0200 Subject: [PATCH 1/4] feat: extensible table --- test/vulpea-db-test.el | 73 +++++++++-------- test/vulpea-perf-test.el | 2 +- vulpea-db.el | 173 ++++++++++++++++++++++++--------------- 3 files changed, 146 insertions(+), 102 deletions(-) diff --git a/test/vulpea-db-test.el b/test/vulpea-db-test.el index efd9c18..3de9ec1 100644 --- a/test/vulpea-db-test.el +++ b/test/vulpea-db-test.el @@ -898,47 +898,50 @@ (org-roam-db-sync 'force) ;; initially there are no vulpea specific tables - (pcase-dolist (`(,table ,_) vulpea-db--schemata) - (expect (org-roam-db-query - [:select name - :from sqlite_master - :where (and (= type 'table) - (= name $r1))] - (emacsql-escape-identifier table)) - :to-equal nil)) - (pcase-dolist (`(,index-name ,_ ,_) vulpea-db--indices) - (expect (org-roam-db-query - [:select name - :from sqlite_master - :where (and (= type 'index) - (= name $r1))] - (emacsql-escape-identifier index-name)) - :to-equal nil)) + (-each vulpea-db--tables + (-lambda ((table _ _ indices)) + (expect (org-roam-db-query + [:select name + :from sqlite_master + :where (and (= type 'table) + (= name $r1))] + (emacsql-escape-identifier table)) + :to-equal nil) + (-each indices + (-lambda ((index-name)) + (expect (org-roam-db-query + [:select name + :from sqlite_master + :where (and (= type 'index) + (= name $r1))] + (emacsql-escape-identifier index-name)) + :to-equal nil))))) ;; then we setup vulpea-db (message "vulpea-db-setup") (vulpea-db-autosync-enable) ;; and vulpea specific tables should exist - (pcase-dolist (`(,table ,_) vulpea-db--schemata) - (expect (org-roam-db-query - [:select name - :from sqlite_master - :where (and (= type 'table) - (= name $r1))] - (emacsql-escape-identifier table)) - :to-equal (list (list (intern (emacsql-escape-identifier table)))))) - (pcase-dolist (`(,index-name ,_ ,_) vulpea-db--indices) - (expect (org-roam-db-query - [:select name - :from sqlite_master - :where (and (= type 'index) - (= name $r1))] - (emacsql-escape-identifier index-name)) - :to-equal (list (list (intern (emacsql-escape-identifier index-name)))))) - (expect (caar (org-roam-db-query [:select version :from cache :where (= id "vulpea")])) - :to-equal - vulpea-db-version) + (-each vulpea-db--tables + (-lambda ((table version _ indices)) + (expect (org-roam-db-query + [:select name + :from sqlite_master + :where (and (= type 'table) + (= name $r1))] + (emacsql-escape-identifier table)) + :to-equal (list (list (intern (emacsql-escape-identifier table))))) + (-each indices + (-lambda ((index-name)) + (expect (org-roam-db-query + [:select name + :from sqlite_master + :where (and (= type 'index) + (= name $r1))] + (emacsql-escape-identifier index-name)) + :to-equal (list (list (intern (emacsql-escape-identifier index-name))))))) + (expect (caar (org-roam-db-query [:select version :from versions :where (= id $s1)] table)) + :to-equal version))) ;; sync a file (message "update file") diff --git a/test/vulpea-perf-test.el b/test/vulpea-perf-test.el index fa6ff82..5d177a4 100644 --- a/test/vulpea-perf-test.el +++ b/test/vulpea-perf-test.el @@ -38,7 +38,7 @@ (require 'org-roam) (require 'vulpea) -(defconst vulpea-perf-zip-branch "attach-dir") +(defconst vulpea-perf-zip-branch "extensible-table") (defconst vulpea-perf-zip-url (format diff --git a/vulpea-db.el b/vulpea-db.el index f2f8f67..8ae5f10 100644 --- a/vulpea-db.el +++ b/vulpea-db.el @@ -344,10 +344,16 @@ If the FILE is relative, it is considered to be relative to -(defconst vulpea-db-version 3) +(defconst vulpea-db-reserved-names + '(notes meta versions + files nodes aliases citations refs tags links) + "List of reserved table names. -(defconst vulpea-db--schemata +Includes Vulpea tables as well as Org roam tables.") + +(defvar vulpea-db--tables '((notes + 1 ([(id :not-null :primary-key) (path :not-null) (level :not-null) @@ -360,6 +366,7 @@ If the FILE is relative, it is considered to be relative to attach] (:foreign-key [path] :references files [file] :on-delete :cascade))) (meta + 1 ([(node-id :not-null) (prop :not-null) (value :not-null)] @@ -368,15 +375,44 @@ If the FILE is relative, it is considered to be relative to :references nodes [id] :on-delete - :cascade))) - (cache + :cascade)) + ((meta-node-id [node-id]))) + (versions + 1 ([(id :not-null :primary-key) - (version :not-null)]))) - "Vulpea db schemata.") - -(defconst vulpea-db--indices - '((meta-node-id meta [node-id])) - "Vulpea db indices.") + (version :not-null)])))) + +(defun vulpea-db-define-table (name version schema &optional indices) + "Define a table with NAME in `org-roam-db'. + +Keep in mind that the names defined in `vulpea-db-reserved-names' +are not allowed. + +VERSION is used to automatically upgrade whenever SCHEMA or +INDICES change. + +A table SCHEMA is a list whose first element is a vector of +column specifications. The rest of the list specifies table +constraints. A column identifier is a symbol and a column's +specification can either be just this symbol or it can include +constraints as a list. Because EmacSQL stores entire Lisp objects +as values, the only relevant (and allowed) types are integer, +float, and object (default). + +Optionally you may define INDICES for this table - an association +list, where car is a unique name of the index and cdr is vector +of columns to be indexed. + +Consult with `emacsql' documentation to learn more about SCHEMA +and INDICES." + (when (seq-contains-p vulpea-db-reserved-names name) + (user-error "Name %s is reserved and can't be used" name)) + (when (seq-contains-p (seq-map #'car vulpea-db--tables) name) + (user-error "Name %s is already in use" name)) + (add-to-list + 'vulpea-db--tables + `(name ,version ,schema ,indices) + 'append)) (defvar vulpea-db--initalized nil "Non-nil when database was initialized.") @@ -388,24 +424,25 @@ GET-DB is a function that returns connection to database." (when-let ((db (funcall get-db))) (unless vulpea-db--initalized (emacsql-with-transaction db - (pcase-dolist (`(,table ,schema) vulpea-db--schemata) - (unless (emacsql db - [:select name - :from sqlite_master - :where (and (= type 'table) - (= name $r1))] - (emacsql-escape-identifier table)) - (emacsql db [:create-table $i1 $S2] table schema))) - (pcase-dolist (`(,index-name ,table ,columns) - vulpea-db--indices) - (unless (emacsql db - [:select name - :from sqlite_master - :where (and (= type 'index) - (= name $r1))] - (emacsql-escape-identifier index-name)) - (emacsql db [:create-index $i1 :on $i2 $S3] - index-name table columns))))) + (-each vulpea-db--tables + (-lambda ((table-name _ schema indices)) + (unless (emacsql db + [:select name + :from sqlite_master + :where (and (= type 'table) + (= name $r1))] + (emacsql-escape-identifier table-name)) + (emacsql db [:create-table $i1 $S2] table-name schema)) + (-each indices + (-lambda ((index-name columns)) + (unless (emacsql db + [:select name + :from sqlite_master + :where (and (= type 'index) + (= name $r1))] + (emacsql-escape-identifier index-name)) + (emacsql db [:create-index $i1 :on $i2 $S3] + index-name table-name columns)))))))) (setq vulpea-db--initalized t) db)) @@ -421,17 +458,13 @@ GET-DB is a function that returns connection to database." (cond (enabled (setq vulpea-db--initalized nil) - ;; attach custom schemata - (seq-each - (lambda (schema) - (add-to-list 'org-roam-db--table-schemata schema 'append)) - vulpea-db--schemata) - - ;; attach custom indices - (seq-each - (lambda (index) - (add-to-list 'org-roam-db--table-indices index 'append)) - vulpea-db--indices) + ;; attach custom schemata and indices + (-each vulpea-db--tables + (-lambda ((table-name _ schema indices)) + (add-to-list 'org-roam-db--table-schemata `(,table-name ,schema) 'append) + (-each indices + (-lambda ((index-name columns)) + (add-to-list 'org-roam-db--table-indices `(,index-name ,table-name ,columns) 'append))))) ;; make sure that extra tables exist table exists (advice-add 'org-roam-db :around #'vulpea-db--init) @@ -449,23 +482,33 @@ GET-DB is a function that returns connection to database." 'org-roam-db-map-links :after #'vulpea-db-insert-links) (when (file-exists-p org-roam-db-location) - (let ((version (or (caar (emacsql (org-roam-db) - [:select version - :from cache - :where (= id "vulpea")])) - 0))) - (when (< version vulpea-db-version) - (org-roam-message (format "Upgrading the vulpea database from version %d to version %d" - version vulpea-db-version)) - (org-roam-db-sync t) - (let ((db (org-roam-db))) - (emacsql db [:update cache - :set (= version $s1) - :where (= id "vulpea")] - vulpea-db-version) - (emacsql db [:insert :or :ignore :into cache [id version] - :values ["vulpea" $s1]] - vulpea-db-version)))))) + (when-let* ((db (org-roam-db)) + (changed (-find + (-lambda ((table-name version1)) + (let ((version0 (or (caar (emacsql + (org-roam-db) + [:select version + :from versions + :where (= id $s1)] + table-name)) + 0))) + (when (< version0 version1) + (org-roam-message + (format + "Doing vulpea database sync to upgrade '%s' table from version %d to version %d" + table-name version0 version1))))) + vulpea-db--tables))) + (org-roam-db-sync t) + (let ((db (org-roam-db))) + (-each vulpea-db--tables + (-lambda ((table-name version)) + (emacsql db [:update versions + :set (= version $s2) + :where (= id $s1)] + table-name version) + (emacsql db [:insert :or :ignore :into versions [id version] + :values [$s1 $s2]] + table-name version))))))) (t (setq vulpea-db--initalized nil) (advice-remove 'org-roam-db-map-links #'vulpea-db-insert-links) @@ -474,16 +517,14 @@ GET-DB is a function that returns connection to database." (advice-remove 'org-roam-db-insert-file-node #'vulpea-db-insert-file-note) (advice-remove 'org-roam-db #'vulpea-db--init) - (seq-each - (lambda (schema) - (setq org-roam-db--table-schemata - (delete schema org-roam-db--table-schemata))) - vulpea-db--schemata) - (seq-each - (lambda (index) - (setq org-roam-db--table-indices - (delete index org-roam-db--table-indices))) - vulpea-db--indices))))) + (-each vulpea-db--tables + (-lambda ((table-name _ schema indices)) + (setq org-roam-db--table-schemata + (delete `(,table-name ,schema) org-roam-db--table-schemata)) + (-each indices + (-lambda ((index-name columns)) + (setq org-roam-db--table-indices + (delete `(,index-name ,table-name ,columns) org-roam-db--table-indices)))))))))) ;;;###autoload (defun vulpea-db-autosync-enable () From 80db75088bc61bfb80cc5790de7f5ab5db97cc8d Mon Sep 17 00:00:00 2001 From: Boris Buliga Date: Fri, 18 Nov 2022 09:30:16 +0200 Subject: [PATCH 2/4] feat: provide a way to reset tables --- vulpea-db.el | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/vulpea-db.el b/vulpea-db.el index 8ae5f10..ecc0bc4 100644 --- a/vulpea-db.el +++ b/vulpea-db.el @@ -351,7 +351,7 @@ If the FILE is relative, it is considered to be relative to Includes Vulpea tables as well as Org roam tables.") -(defvar vulpea-db--tables +(defvar vulpea-db--tables-default '((notes 1 ([(id :not-null :primary-key) @@ -382,6 +382,15 @@ Includes Vulpea tables as well as Org roam tables.") ([(id :not-null :primary-key) (version :not-null)])))) +(defvar vulpea-db--tables vulpea-db--tables-default) + +(defvar vulpea-db--initalized nil + "Non-nil when database was initialized.") + +(defun vulpea-db-reset-tables () + "Reset defined tables." + (setq vulpea-db--tables vulpea-db--tables-default)) + (defun vulpea-db-define-table (name version schema &optional indices) "Define a table with NAME in `org-roam-db'. @@ -411,11 +420,9 @@ and INDICES." (user-error "Name %s is already in use" name)) (add-to-list 'vulpea-db--tables - `(name ,version ,schema ,indices) - 'append)) - -(defvar vulpea-db--initalized nil - "Non-nil when database was initialized.") + `(,name ,version ,schema ,indices) + 'append) + (setq vulpea-db--initalized nil)) (defun vulpea-db--init (get-db) "Initialize database by creating missing tables if needed. From 7caf1d6bb4985ae108e77b27b0785df6d6251882 Mon Sep 17 00:00:00 2001 From: Boris Buliga Date: Fri, 18 Nov 2022 09:31:19 +0200 Subject: [PATCH 3/4] feat: provide a hook for db insertion --- test/vulpea-db-test.el | 67 +++++++++++++++++++++++++++++++++++++++ vulpea-db.el | 72 ++++++++++++++++++++++++++++++++---------- 2 files changed, 123 insertions(+), 16 deletions(-) diff --git a/test/vulpea-db-test.el b/test/vulpea-db-test.el index 3de9ec1..7f476ff 100644 --- a/test/vulpea-db-test.el +++ b/test/vulpea-db-test.el @@ -978,5 +978,72 @@ ("answer" . ("42"))) :attach-dir (expand-file-name "data/05/907606-f836-45bf-bd36-a8444308eddd" org-roam-directory))))) +(describe "vulpea-db-insert-note-functions" + (before-each + (vulpea-test--init) + (spy-on 'insert-handle-fn) + (add-hook 'vulpea-db-insert-note-functions #'insert-handle-fn)) + + (after-each + (vulpea-test--teardown) + (remove-hook 'vulpea-db-insert-note-functions #'insert-handle-fn)) + + (it "calls insert hook on file level note" + (let* ((id "1cc15044-aedb-442e-b727-9e3f7346be95") + (note (vulpea-db-get-by-id id)) + (file (vulpea-note-path note))) + ;; force sync of a single file by clearing it and syncing manually + (org-roam-db-clear-file file) + (org-roam-db-sync) + (expect 'insert-handle-fn :to-have-been-called-times 1) + (expect 'insert-handle-fn :to-have-been-called-with + (make-vulpea-note + :path (expand-file-name "note-with-link.org" org-roam-directory) + :title "Note with link" + :tags nil + :level 0 + :id "1cc15044-aedb-442e-b727-9e3f7346be95" + :links nil ; not supported in this hook + :properties (list + (cons "CATEGORY" "note-with-link") + (cons "ID" "1cc15044-aedb-442e-b727-9e3f7346be95") + (cons "BLOCKED" "") + (cons "FILE" (expand-file-name "note-with-link.org" org-roam-directory)) + (cons "PRIORITY" "B")) + :attach-dir (expand-file-name "data/1c/c15044-aedb-442e-b727-9e3f7346be95" org-roam-directory))))) + + (it "calls insert hook on outline level note" + (let* ((id "1cc15044-aedb-442e-b727-9e3f7346be95")) + ;; force sync by adding a new header to the end of existing note + (vulpea-utils-with-note (vulpea-db-get-by-id id) + (goto-char (point-max)) + (insert + "\n" + "* I was added\n") + (setf id (org-id-get-create)) + (save-buffer)) + (expect 'insert-handle-fn :to-have-been-called-times 2) + (expect 'insert-handle-fn :to-have-been-called-with + (make-vulpea-note + :path (expand-file-name "note-with-link.org" org-roam-directory) + :title "I was added" + :tags nil + :level 1 + :id id + :links nil ; not supported in this hook + :properties (list + (cons "CATEGORY" "note-with-link") + (cons "ID" id) + (cons "BLOCKED" "") + (cons "FILE" (expand-file-name "note-with-link.org" org-roam-directory)) + (cons "PRIORITY" "B") + (cons "ITEM" "I was added")) + :attach-dir (expand-file-name + (concat "data/" + (substring-no-properties id 0 2) + "/" + (substring-no-properties id 2)) + org-roam-directory)))))) + (provide 'vulpea-db-test) ;;; vulpea-db-test.el ends here diff --git a/vulpea-db.el b/vulpea-db.el index ecc0bc4..54d8cad 100644 --- a/vulpea-db.el +++ b/vulpea-db.el @@ -344,6 +344,26 @@ If the FILE is relative, it is considered to be relative to +(defvar vulpea-db-insert-note-functions nil + "Abnormal hooks to run after a `vulpea-note' is inserted to DB. + +The hook is called with a single argument - an inserted +`vulpea-note'. Keep in mind that due to Org Roam implementation +details, links are not present in passed `vulpea-note'. + +Use it to update any custom tables you added via +`vulpea-db-define-table'. Keep in mind that you should not expect +any entries to be present in database. The synchronisation order +is _not_ defined. + +There is no similar hook for removal, because it can be handled +by using combination of :foreign-key and :on-delete :cascade +options in table schema. See `vulpea-db--tables-default' for +example. + +Each function accepts a note that was inserted via +`vulpea-insert'.") + (defconst vulpea-db-reserved-names '(notes meta versions files nodes aliases citations refs tags links) @@ -477,16 +497,9 @@ GET-DB is a function that returns connection to database." (advice-add 'org-roam-db :around #'vulpea-db--init) ;; make sure that all data is inserted into table - (advice-add - 'org-roam-db-insert-file-node - :after - #'vulpea-db-insert-file-note) - (advice-add - 'org-roam-db-insert-node-data - :after - #'vulpea-db-insert-outline-note) - (advice-add - 'org-roam-db-map-links :after #'vulpea-db-insert-links) + (advice-add 'org-roam-db-insert-file-node :after #'vulpea-db-insert-file-note) + (advice-add 'org-roam-db-insert-node-data :after #'vulpea-db-insert-outline-note) + (advice-add 'org-roam-db-map-links :after #'vulpea-db-insert-links) (when (file-exists-p org-roam-db-location) (when-let* ((db (org-roam-db)) @@ -519,10 +532,8 @@ GET-DB is a function that returns connection to database." (t (setq vulpea-db--initalized nil) (advice-remove 'org-roam-db-map-links #'vulpea-db-insert-links) - (advice-remove - 'org-roam-db-insert-node-data #'vulpea-db-insert-outline-note) - (advice-remove - 'org-roam-db-insert-file-node #'vulpea-db-insert-file-note) + (advice-remove 'org-roam-db-insert-node-data #'vulpea-db-insert-outline-note) + (advice-remove 'org-roam-db-insert-file-node #'vulpea-db-insert-file-note) (advice-remove 'org-roam-db #'vulpea-db--init) (-each vulpea-db--tables (-lambda ((table-name _ schema indices)) @@ -630,7 +641,23 @@ GET-DB is a function that returns connection to database." (seq-map (lambda (kvp) (vector id (car kvp) (cdr kvp))) - kvps)))))))) + kvps))) + (run-hook-with-args + 'vulpea-db-insert-note-functions + (make-vulpea-note + :id id + :path file + :level level + :title title + :aliases aliases + :tags tags + :links nil + :properties properties + :meta (seq-map + (lambda (kvp) + (cons (car kvp) (cdr kvp))) + kvps) + :attach-dir attach))))))) (defun vulpea-db-insert-outline-note () "Insert outline level note into `vulpea' database." @@ -675,7 +702,20 @@ GET-DB is a function that returns connection to database." tags nil nil - attach))))) + attach)) + (run-hook-with-args + 'vulpea-db-insert-note-functions + (make-vulpea-note + :id id + :path file + :level level + :title title + :aliases aliases + :tags tags + :links nil + :properties properties + :meta nil + :attach-dir attach))))) (defun vulpea-db-insert-links (&rest _) "Insert links into `vulpea' database." From a83fd16864e10cc0e77aa1028a3c3486fd6ffc7d Mon Sep 17 00:00:00 2001 From: Boris Buliga Date: Mon, 21 Nov 2022 13:15:57 +0200 Subject: [PATCH 4/4] docs: document table extensibility --- CHANGELOG.org | 1 + README.org | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/CHANGELOG.org b/CHANGELOG.org index 9e74779..dd22bd2 100644 --- a/CHANGELOG.org +++ b/CHANGELOG.org @@ -14,6 +14,7 @@ - Include attachment directory in =vulpea-note= and =notes= table. - [[https://github.com/d12frosted/vulpea/issues/130][vulpea#130]] Introduce mechanism to automatically rebuild database when needed. This happens either on the first usage of Vulpea or when =vulpea-db-version= increases. +- [[https://github.com/d12frosted/vulpea/pull/158][vulpea#158]] Introduce mechanism to define more tables in Org Roam database (see =vulpea-db-define-table=). And provide a hook to fill defined tables with data (see =vulpea-db-insert-note-functions=). ** v0.3.0 :PROPERTIES: diff --git a/README.org b/README.org index e712a36..0da04b9 100644 --- a/README.org +++ b/README.org @@ -1041,6 +1041,33 @@ As you can see, =vulpea-db-query= doesn't allow to pass any custom SQL for filte - =vulpea-db-get-file-by-id= - function to get =FILE= of a note represented by =ID=. Supports headings of the note. - =vulpea-db-search-by-title= - function to query notes with =TITLE=. +**** Extending database +:PROPERTIES: +:ID: 8d0b72a8-b5ee-4be8-a17f-84b151ad85fc +:END: + +You may extend database by adding custom tables using =vulpea-db-define-table=: + +#+begin_src emacs-lisp + (vulpea-db-define-table + ;; name + 'my-custom-table + ;; version + 1 + ;; schema + '([(note-id :unique :primary-key) + (some-column :not-null) + (some-other-column)] + ;; useful to automatically cleanup your table whenever a note/node/file is removed + (:foreign-key [note-id] :references nodes [id] :on-delete :cascade)) + ;; index + '((custom-node-id-index [note-id]))) +#+end_src + +Consult with [[https://github.com/magit/emacsql/][magit/emacsql]] documentation to learn more about schema and indices. + +In order to populate your table with data, you should add a hook to =vulpea-db-insert-note-functions=. It is called with a single argument of type =vulpea-note= (keep in mind that =vulpea-note-links= slot is empty, open a ticket if you need it). + *** =vulpea-meta= :PROPERTIES: :ID: 9bb0311f-c257-46f1-8e1f-68c735a1a07c