Merge pull request #738 from samply/db-versioning

Introduce Database Versioning

docs/database/migration.md

@@ -0,0 +1,61 @@
+# Database Migration
+
+If Blaze starts up with the following error message:
+
+```
+Incompatible index store version <x> found. This version of Blaze needs
+version <y>.
+
+Either use an older version of Blaze which is compatible with index store
+version <x> or do a database migration described here:
+
+  https://github.com/samply/blaze/tree/master/docs/database/migration.md
+```
+
+you need to do a migration of the index store.
+
+Blaze will do that migration automatically at startup if you delete the index store. So all you need to do is to make a
+backup of all the data Blaze has written to disk, **plan for a downtime**, delete the index store and start Blaze normally.
+
+## Deleting the Index Store
+
+Please start Blaze with a shell assuming that you use the volume `blaze-data`:
+
+```sh
+docker run -it -v blaze-data:/app/data samply/blaze:0.17 sh
+```
+
+in that shell, go into `/app/data` and list all directories:
+
+```sh
+cd /app/data
+ls
+```
+
+you should see the three directories `index`,  `resource` and `transaction`.
+
+If you have enough disk space, just rename the index directory into `index-old`. If not, delete it assuming you have a
+backup!
+
+Exit the shell und start Blaze normally.
+
+## Index Store Migration at Start
+
+If you start Blaze without an index store, it will use the transaction log and the resource store to recreate the index
+store. During that process Blaze will not be available for reads and writes. Reading from Blaze will result in
+a `503 Service Unavailable` with the following `OperationOutcome`:
+
+```json
+{
+  "resourceType": "OperationOutcome",
+  "issue": [
+    {
+      "severity": "error",
+      "code": "timeout",
+      "diagnostics": "Timeout while trying to acquire the latest known database state. At least one known transaction hasn't been completed yet. Please try to lower the transaction load or increase the timeout of 10000 ms by setting DB_SYNC_TIMEOUT to a higher value if you see this often."
+    }
+  ]
+}
+```
+
+The time needed for the rebuild of the index store is roughly the time all transactions ever happened in this instance of Blaze took.

modules/db/src/blaze/db/node.clj

@@ -22,6 +22,7 @@
     [blaze.db.node.tx-indexer :as tx-indexer]
     [blaze.db.node.tx-indexer.verify :as tx-indexer-verify]
     [blaze.db.node.validation :as validation]
+    [blaze.db.node.version :as version]
     [blaze.db.resource-store :as rs]
     [blaze.db.search-param-registry.spec]
     [blaze.db.tx-log :as tx-log]
@@ -387,12 +388,46 @@
     [:blaze.db/enforce-referential-integrity]))
 
 
+(def ^:private expected-kv-store-version 0)
+
+
+(defn- kv-store-version [kv-store]
+  (or (some-> (kv/get kv-store version/key) version/decode-value) 0))
+
+
+(def ^:private incompatible-kv-store-version-msg
+  "Incompatible index store version %1$d found. This version of Blaze needs
+  version %2$d.
+
+  Either use an older version of Blaze which is compatible with index store
+  version %1$d or do a database migration described here:
+
+    https://github.com/samply/blaze/tree/master/docs/database/migration.md
+
+  ")
+
+
+(defn- incompatible-kv-store-version-ex [actual-version expected-version]
+  (ex-info (format incompatible-kv-store-version-msg actual-version expected-version)
+           {:actual-version actual-version :expected-version expected-version}))
+
+
+(defn- check-version! [kv-store]
+  (when (tx-success/last-t kv-store)
+    (let [actual-kv-store-version (kv-store-version kv-store)]
+      (if (= actual-kv-store-version expected-kv-store-version)
+        (log/info "Index store version is" actual-kv-store-version)
+        (throw (incompatible-kv-store-version-ex actual-kv-store-version
+                                                 expected-kv-store-version))))))
+
+
 (defmethod ig/init-key :blaze.db/node
   [_ {:keys [tx-log resource-handle-cache tx-cache indexer-executor kv-store
              resource-indexer resource-store search-param-registry poll-timeout]
       :or {poll-timeout (time/seconds 1)}
       :as config}]
   (init-msg config)
+  (check-version! kv-store)
   (let [node (->Node (ctx config) tx-log resource-handle-cache tx-cache kv-store
                      resource-store search-param-registry resource-indexer
                      (atom (initial-state kv-store))

modules/db/src/blaze/db/node/version.clj

@@ -0,0 +1,23 @@
+(ns blaze.db.node.version
+  (:refer-clojure :exclude [key])
+  (:require
+    [blaze.byte-buffer :as bb])
+  (:import
+    [java.nio.charset StandardCharsets]))
+
+
+(set! *warn-on-reflection* true)
+
+
+(def key
+  (.getBytes "version" StandardCharsets/ISO_8859_1))
+
+
+(defn encode-value [version]
+  (-> (bb/allocate Integer/BYTES)
+      (bb/put-int! version)
+      (bb/array)))
+
+
+(defn decode-value [bytes]
+  (bb/get-int! (bb/wrap bytes)))

modules/db/test/blaze/db/node_test.clj

@@ -6,11 +6,14 @@
     [blaze.db.api :as d]
     [blaze.db.api-spec]
     [blaze.db.impl.db-spec]
+    [blaze.db.impl.index.tx-success :as tx-success]
+    [blaze.db.kv :as kv]
     [blaze.db.kv.mem-spec]
     [blaze.db.node :as node]
     [blaze.db.node-spec]
     [blaze.db.node.resource-indexer :as resource-indexer]
     [blaze.db.node.tx-indexer :as-alias tx-indexer]
+    [blaze.db.node.version :as version]
     [blaze.db.resource-handle-cache]
     [blaze.db.resource-store :as rs]
     [blaze.db.search-param-registry]
@@ -29,7 +32,8 @@
     [cognitect.anomalies :as anom]
     [integrant.core :as ig]
     [juxt.iota :refer [given]]
-    [taoensso.timbre :as log]))
+    [taoensso.timbre :as log])
+  (:import [java.time Instant]))
 
 
 (set! *warn-on-reflection* true)
@@ -82,6 +86,12 @@
              {:resource-store (ig/ref ::rs/kv)})))
 
 
+(defn- with-index-store-version [system version]
+  (assoc-in system [[::kv/mem :blaze.db/index-kv-store] :init-data]
+            [[version/key (version/encode-value version)]
+             (tx-success/index-entry 1 Instant/EPOCH)]))
+
+
 (deftest init-test
   (testing "nil config"
     (given-thrown (ig/init {:blaze.db/node nil})
@@ -129,7 +139,14 @@
       [:explain ::s/problems 6 :pred] := `(fn ~'[%] (contains? ~'% :resource-store))
       [:explain ::s/problems 7 :pred] := `(fn ~'[%] (contains? ~'% :search-param-registry))
       [:explain ::s/problems 8 :pred] := `boolean?
-      [:explain ::s/problems 8 :val] := ::invalid)))
+      [:explain ::s/problems 8 :val] := ::invalid))
+
+  (testing "incompatible version"
+    (given-thrown (ig/init (with-index-store-version system -1))
+      :key := :blaze.db/node
+      :reason := ::ig/build-threw-exception
+      [:cause-data :expected-version] := 0
+      [:cause-data :actual-version] := -1)))
 
 
 (deftest duration-seconds-collector-init-test
@@ -220,3 +237,8 @@
 
     ;; but it isn't terminated yet
     (is (not (ex/terminated? indexer-executor)))))
+
+
+(deftest existing-data-with-compatible-version
+  (with-system [{:blaze.db/keys [node]} (with-index-store-version system 0)]
+    (is node)))

modules/rest-util/src/blaze/middleware/fhir/db.clj

@@ -17,7 +17,7 @@
 
 
 (defn- timeout-msg [timeout]
-  (format "Timeout while trying to acquire the latest known database state. At least one known transaction hasen't been completed yet. Please try to lower the transaction load or increase the timeout of %d ms by setting DB_SYNC_TIMEOUT to a higher value if you see this often.", timeout))
+  (format "Timeout while trying to acquire the latest known database state. At least one known transaction hasn't been completed yet. Please try to lower the transaction load or increase the timeout of %d ms by setting DB_SYNC_TIMEOUT to a higher value if you see this often.", timeout))
 
 
 (defn- db [node timeout {:keys [query-params] :as request}]

modules/rest-util/test/blaze/middleware/fhir/db_test.clj

@@ -88,7 +88,7 @@
 
         (given-failed-future ((db/wrap-db handler ::node 2000) {})
           ::anom/category := ::anom/busy
-          ::anom/message := "Timeout while trying to acquire the latest known database state. At least one known transaction hasen't been completed yet. Please try to lower the transaction load or increase the timeout of 2000 ms by setting DB_SYNC_TIMEOUT to a higher value if you see this often.")))
+          ::anom/message := "Timeout while trying to acquire the latest known database state. At least one known transaction hasn't been completed yet. Please try to lower the transaction load or increase the timeout of 2000 ms by setting DB_SYNC_TIMEOUT to a higher value if you see this often.")))
 
     (testing "default timeout are 10 seconds"
       (with-redefs
@@ -99,4 +99,4 @@
 
         (given-failed-future ((db/wrap-db handler ::node) {})
           ::anom/category := ::anom/busy
-          ::anom/message := "Timeout while trying to acquire the latest known database state. At least one known transaction hasen't been completed yet. Please try to lower the transaction load or increase the timeout of 10000 ms by setting DB_SYNC_TIMEOUT to a higher value if you see this often.")))))
+          ::anom/message := "Timeout while trying to acquire the latest known database state. At least one known transaction hasn't been completed yet. Please try to lower the transaction load or increase the timeout of 10000 ms by setting DB_SYNC_TIMEOUT to a higher value if you see this often.")))))

modules/test-util/src/blaze/test_util.clj

@@ -17,8 +17,14 @@
 (set! *warn-on-reflection* true)
 
 
+(defn all-ex-data [e]
+  (cond-> (ex-data e)
+    (ex-data (ex-cause e))
+    (assoc :cause-data (all-ex-data (ex-cause e)))))
+
+
 (defmacro given-thrown [v & body]
-  `(given (try ~v (is false) (catch Exception e# (ex-data e#)))
+  `(given (try ~v (is false) (catch Exception e# (all-ex-data e#)))
      ~@body))