port kvstore from nim-beacon-chain (#230)

* port kvstore from nim-beacon-chain
* remove old database backends
* use kvstore in trie database
* add sqlite dep
* avoid template param double evaluation
* clean up heterogenous lookup todo

eth.nimble

@@ -13,29 +13,22 @@ requires "nim >= 1.2.0",
          "chronicles",
          "stew",
          "nat_traversal",
-         "metrics"
+         "metrics",
+         "sqlite3_abi"
 
 proc runTest(path: string) =
   echo "\nRunning: ", path
   exec "nim c -r -d:release -d:chronicles_log_level=ERROR --verbosity:0 --hints:off " & path
   rmFile path
 
 proc runKeyfileTests() =
-  for filename in [
-      "test_keyfile",
-      "test_uuid",
-    ]:
-    runTest("tests/keyfile/" & filename)
+  runTest("tests/keyfile/all_tests")
 
 task test_keyfile, "run keyfile tests":
   runKeyfileTests()
 
 proc runKeysTests() =
-  for filename in [
-      "test_keys",
-      "test_private_public_key_consistency"
-    ]:
-    runTest("tests/keys/" & filename)
+  runTest("tests/keys/all_tests")
 
 task test_keys, "run keys tests":
   runKeysTests()
@@ -77,10 +70,15 @@ proc runTrieTests() =
 task test_trie, "run trie tests":
   runTrieTests()
 
+proc runDbTests() =
+  runTest("tests/db/all_tests")
+
+task test_db, "run db tests":
+  runDbTests()
+
 task test, "run tests":
   for filename in [
       "test_bloom",
-      "test_common",
     ]:
     runTest("tests/" & filename)
 
@@ -89,4 +87,4 @@ task test, "run tests":
   runP2pTests()
   runRlpTests()
   runTrieTests()
-
+  runDbTests()

eth/db/kvstore.nim

@@ -0,0 +1,114 @@
+# beacon_chain
+# Copyright (c) 2018-2020 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+## Simple Key-Value store database interface that allows creating multiple
+## tables within each store
+
+{.push raises: [Defect].}
+
+import
+  stew/results,
+  tables, hashes, sets
+
+export results
+
+type
+  MemStoreRef* = ref object of RootObj
+    records: Table[seq[byte], seq[byte]]
+      # TODO interaction with this table would benefit from heterogenous lookup
+      #      (see `@key` below)
+      #      https://github.com/nim-lang/Nim/issues/7457
+
+  KvResult*[T] = Result[T, string]
+
+  DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [Defect].}
+
+  PutProc = proc (db: RootRef, key, val: openArray[byte]): KvResult[void] {.nimcall, gcsafe, raises: [Defect].}
+  GetProc = proc (db: RootRef, key: openArray[byte], onData: DataProc): KvResult[bool] {.nimcall, gcsafe, raises: [Defect].}
+  DelProc = proc (db: RootRef, key: openArray[byte]): KvResult[void] {.nimcall, gcsafe, raises: [Defect].}
+  ContainsProc = proc (db: RootRef, key: openArray[byte]): KvResult[bool] {.nimcall, gcsafe, raises: [Defect].}
+
+  KvStoreRef* = ref object
+    ## Key-Value store virtual interface
+    obj: RootRef
+    putProc: PutProc
+    getProc: GetProc
+    delProc: DelProc
+    containsProc: ContainsProc
+
+template put*(dbParam: KvStoreRef, key, val: openArray[byte]): KvResult[void] =
+  ## Store ``value`` at ``key`` - overwrites existing value if already present
+  let db = dbParam
+  db.putProc(db.obj, key, val)
+
+template get*(dbParam: KvStoreRef, key: openArray[byte], onData: untyped): KvResult[bool] =
+  ## Retrive value at ``key`` and call ``onData`` with the value. The data is
+  ## valid for the duration of the callback.
+  ## ``onData``: ``proc(data: openArray[byte])``
+  ## returns true if found and false otherwise.
+  let db = dbParam
+  db.getProc(db.obj, key, onData)
+
+template del*(dbParam: KvStoreRef, key: openArray[byte]): KvResult[void] =
+  ## Remove value at ``key`` from store - do nothing if the value is not present
+  let db = dbParam
+  db.delProc(db.obj, key)
+
+template contains*(dbParam: KvStoreRef, key: openArray[byte]): KvResult[bool] =
+  ## Return true iff ``key`` has a value in store
+  let db = dbParam
+  db.containsProc(db.obj, key)
+
+proc putImpl[T](db: RootRef, key, val: openArray[byte]): KvResult[void] =
+  mixin put
+  put(T(db), key, val)
+
+proc getImpl[T](db: RootRef, key: openArray[byte], onData: DataProc): KvResult[bool] =
+  mixin get
+  get(T(db), key, onData)
+
+proc delImpl[T](db: RootRef, key: openArray[byte]): KvResult[void] =
+  mixin del
+  del(T(db), key)
+
+proc containsImpl[T](db: RootRef, key: openArray[byte]): KvResult[bool] =
+  mixin contains
+  contains(T(db), key)
+
+func kvStore*[T: RootRef](x: T): KvStoreRef =
+  mixin del, get, put, contains
+
+  KvStoreRef(
+    obj: x,
+    putProc: putImpl[T],
+    getProc: getImpl[T],
+    delProc: delImpl[T],
+    containsProc: containsImpl[T]
+  )
+
+proc get*(db: MemStoreRef, key: openArray[byte], onData: DataProc): KvResult[bool] =
+  db.records.withValue(@key, v):
+    onData(v[])
+    return ok(true)
+
+  ok(false)
+
+proc del*(db: MemStoreRef, key: openArray[byte]): KvResult[void] =
+  db.records.del(@key)
+  ok()
+
+proc contains*(db: MemStoreRef, key: openArray[byte]): KvResult[bool] =
+  ok(db.records.contains(@key))
+
+proc put*(db: MemStoreRef, key, val: openArray[byte]): KvResult[void] =
+  db.records[@key] = @val
+  ok()
+
+proc init*(T: type MemStoreRef): T =
+  T(
+    records: initTable[seq[byte], seq[byte]]()
+  )

eth/db/kvstore_rocksdb.nim

@@ -0,0 +1,46 @@
+{.push raises: [Defect].}
+
+import os, rocksdb, ./kvstore, stew/results
+
+export results
+
+const maxOpenFiles = 512
+
+type
+  RocksStoreRef* = ref object of RootObj
+    store: RocksDBInstance
+
+proc get*(db: RocksStoreRef, key: openarray[byte], onData: kvstore.DataProc): KvResult[bool] =
+  db.store.get(key, onData)
+
+proc put*(db: RocksStoreRef, key, value: openarray[byte]): KvResult[void] =
+  db.store.put(key, value)
+
+proc contains*(db: RocksStoreRef, key: openarray[byte]): KvResult[bool] =
+  db.store.contains(key)
+
+proc del*(db: RocksStoreRef, key: openarray[byte]): KvResult[void] =
+  db.store.del(key)
+
+proc close*(db: RocksStoreRef) =
+  db.store.close
+
+proc init*(
+    T: type RocksStoreRef, basePath: string, name: string,
+    readOnly = false): KvResult[T] =
+  let
+    dataDir = basePath / name / "data"
+    backupsDir = basePath / name / "backups"
+
+  try:
+    createDir(dataDir)
+    createDir(backupsDir)
+  except OSError, IOError:
+    return err("rocksdb: cannot create database directory")
+
+  var store: RocksDBInstance
+  if (let v = store.init(
+      dataDir, backupsDir, readOnly, maxOpenFiles = maxOpenFiles); v.isErr):
+    return err(v.error)
+
+  ok(T(store: store))

eth/db/kvstore_sqlite3.nim

@@ -0,0 +1,158 @@
+## Implementation of KvStore based on sqlite3
+
+{.push raises: [Defect].}
+
+import
+  os,
+  sqlite3_abi,
+  ./kvstore
+
+export kvstore
+
+type
+  SqStoreRef* = ref object of RootObj
+    env: ptr sqlite3
+    selectStmt, insertStmt, deleteStmt: ptr sqlite3_stmt
+
+template checkErr(op, cleanup: untyped) =
+  if (let v = (op); v != SQLITE_OK):
+    cleanup
+    return err($sqlite3_errstr(v))
+
+template checkErr(op) =
+  checkErr(op): discard
+
+proc bindBlob(s: ptr sqlite3_stmt, n: int, blob: openarray[byte]): cint =
+  sqlite3_bind_blob(s, n.cint, unsafeAddr blob[0], blob.len.cint, nil)
+
+proc get*(db: SqStoreRef, key: openarray[byte], onData: DataProc): KvResult[bool] =
+  checkErr sqlite3_reset(db.selectStmt)
+  checkErr sqlite3_clear_bindings(db.selectStmt)
+  checkErr bindBlob(db.selectStmt, 1, key)
+
+  let v = sqlite3_step(db.selectStmt)
+  case v
+  of SQLITE_ROW:
+    let
+      p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(db.selectStmt, 0))
+      l = sqlite3_column_bytes(db.selectStmt, 0)
+    onData(toOpenArray(p, 0, l-1))
+    ok(true)
+  of SQLITE_DONE:
+    ok(false)
+  else:
+    err($sqlite3_errstr(v))
+
+proc put*(db: SqStoreRef, key, value: openarray[byte]): KvResult[void] =
+  checkErr sqlite3_reset(db.insertStmt)
+  checkErr sqlite3_clear_bindings(db.insertStmt)
+
+  checkErr bindBlob(db.insertStmt, 1, key)
+  checkErr bindBlob(db.insertStmt, 2, value)
+
+  if (let v = sqlite3_step(db.insertStmt); v != SQLITE_DONE):
+    err($sqlite3_errstr(v))
+  else:
+    ok()
+
+proc contains*(db: SqStoreRef, key: openarray[byte]): KvResult[bool] =
+  checkErr sqlite3_reset(db.selectStmt)
+  checkErr sqlite3_clear_bindings(db.selectStmt)
+
+  checkErr bindBlob(db.selectStmt, 1, key)
+
+  let v = sqlite3_step(db.selectStmt)
+  case v
+  of SQLITE_ROW: ok(true)
+  of SQLITE_DONE: ok(false)
+  else: err($sqlite3_errstr(v))
+
+proc del*(db: SqStoreRef, key: openarray[byte]): KvResult[void] =
+  checkErr sqlite3_reset(db.deleteStmt)
+  checkErr sqlite3_clear_bindings(db.deleteStmt)
+
+  checkErr bindBlob(db.deleteStmt, 1, key)
+
+  if (let v = sqlite3_step(db.deleteStmt); v != SQLITE_DONE):
+    err($sqlite3_errstr(v))
+  else:
+    ok()
+
+proc close*(db: SqStoreRef) =
+  discard sqlite3_finalize(db.insertStmt)
+  discard sqlite3_finalize(db.selectStmt)
+  discard sqlite3_finalize(db.deleteStmt)
+
+  discard sqlite3_close(db.env)
+
+  db[] = SqStoreRef()[]
+
+proc init*(
+    T: type SqStoreRef,
+    basePath: string,
+    name: string,
+    readOnly = false,
+    inMemory = false): KvResult[T] =
+  var
+    env: ptr sqlite3
+
+  let
+    name =
+      if inMemory: ":memory:"
+      else: basepath / name & ".sqlite3"
+    flags =
+      if readOnly: SQLITE_OPEN_READONLY
+      else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE
+
+  if not inMemory:
+    try:
+      createDir(basePath)
+    except OSError, IOError:
+      return err("`sqlite: cannot create database directory")
+
+  checkErr sqlite3_open_v2(name, addr env, flags.cint, nil)
+
+  template prepare(q: string, cleanup: untyped): ptr sqlite3_stmt =
+    var s: ptr sqlite3_stmt
+    checkErr sqlite3_prepare_v2(env, q, q.len.cint, addr s, nil):
+      cleanup
+      discard sqlite3_close(env)
+    s
+
+  template checkExec(q: string) =
+    let s = prepare(q): discard
+
+    if (let x = sqlite3_step(s); x != SQLITE_DONE):
+      discard sqlite3_finalize(s)
+      discard sqlite3_close(env)
+      return err($sqlite3_errstr(x))
+
+    if (let x = sqlite3_finalize(s); x != SQLITE_OK):
+      discard sqlite3_close(env)
+      return err($sqlite3_errstr(x))
+
+  # TODO: check current version and implement schema versioning
+  checkExec "PRAGMA user_version = 1;"
+
+  checkExec """
+    CREATE TABLE IF NOT EXISTS kvstore(
+       key BLOB PRIMARY KEY,
+       value BLOB
+    ) WITHOUT ROWID;
+  """
+
+  let
+    selectStmt = prepare "SELECT value FROM kvstore WHERE key = ?;":
+      discard
+    insertStmt = prepare "INSERT OR REPLACE INTO kvstore(key, value) VALUES (?, ?);":
+      discard sqlite3_finalize(selectStmt)
+    deleteStmt = prepare "DELETE FROM kvstore WHERE key = ?;":
+      discard sqlite3_finalize(selectStmt)
+      discard sqlite3_finalize(insertStmt)
+
+  ok(SqStoreRef(
+    env: env,
+    selectStmt: selectStmt,
+    insertStmt: insertStmt,
+    deleteStmt: deleteStmt
+  ))