From 263d9a38cccce88b67fc85f579ab6cc1e541a4a2 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 12:26:56 +0300 Subject: [PATCH 01/26] Add pool implementation for SQLite. --- src/norm/pool.nim | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/norm/pool.nim diff --git a/src/norm/pool.nim b/src/norm/pool.nim new file mode 100644 index 00000000..574278b4 --- /dev/null +++ b/src/norm/pool.nim @@ -0,0 +1,69 @@ +import std/locks + +import sqlite + + +type + Pool* = ref object + defaultSize: Natural + conns: seq[DbConn] + poolExhaustedPolicy: PoolExhaustedPolicy + lock: Lock + PoolExhaustedError* = object of CatchableError + PoolExhaustedPolicy* = enum + pepRaise + pepExtend + + +func newPool*(defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool = + result = Pool(defaultSize: defaultSize, conns: newSeq[DbConn](defaultSize), poolExhaustedPolicy: poolExhaustedPolicy) + + initLock(result.lock) + + for conn in result.conns.mitems: + conn = getDb() + +func defaultSize*(pool: Pool): Natural = + pool.defaultSize + +func size*(pool: Pool): Natural = + len(pool.conns) + +func pop*(pool: var Pool): DbConn = + withLock(pool.lock): + if pool.size > 0: + result = pool.conns.pop() + else: + case pool.poolExhaustedPolicy + of pepRaise: + raise newException(PoolExhaustedError, "Pool exhausted") + of pepExtend: + result = getDb() + +func add*(pool: var Pool, dbConn: DbConn) = + withLock(pool.lock): + pool.conns.add(dbConn) + +func reset*(pool: var Pool) = + withLock(pool.lock): + while pool.size > pool.defaultSize: + var conn = pool.conns.pop() + close conn + +func close*(pool: var Pool) = + withLock(pool.lock): + for conn in pool.conns.mitems: + close conn + + pool.conns.setLen(0) + +template withDb*(pool: var Pool, body: untyped): untyped = + block: + let db {.inject.} = pool.pop() + + try: + body + + finally: + pool.add(db) + From 087c32448c666754f6a055aca7060b9d8ce3bb25 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 12:27:19 +0300 Subject: [PATCH 02/26] Add tests for connection pool. --- tests/sqlite/tpool.nim | 161 ++++++++++++++++++++++++++++++++++++++++ tests/sqlite/tpool.nims | 2 + 2 files changed, 163 insertions(+) create mode 100644 tests/sqlite/tpool.nim create mode 100644 tests/sqlite/tpool.nims diff --git a/tests/sqlite/tpool.nim b/tests/sqlite/tpool.nim new file mode 100644 index 00000000..3c800326 --- /dev/null +++ b/tests/sqlite/tpool.nim @@ -0,0 +1,161 @@ +import std/[unittest, os, strutils] + +import norm/[sqlite, pool] + +import ../models + +const dbFile = "test.db" + +suite "Connection pool": + setup: + removeFile dbFile + putEnv(dbHostEnv, dbFile) + + teardown: + delEnv(dbHostEnv) + removeFile dbFile + + test "Create and close pool": + var pool = newPool(1) + + check pool.defaultSize == 1 + check pool.size == 1 + + close pool + + check pool.size == 0 + + test "Explicit pool connection": + var pool = newPool(1) + let db = pool.pop() + + db.createTables(newToy()) + + let qry = "PRAGMA table_info($#);" + + check db.getAllRows(sql qry % "Toy") == @[ + @[?0, ?"price", ?"FLOAT", ?1, ?nil, ?0], + @[?1, ?"id", ?"INTEGER", ?1, ?nil, ?1], + ] + + pool.add(db) + close pool + + test "Implicit pool connection": + var pool = newPool(1) + + withDb(pool): + db.createTables(newToy()) + + let qry = "PRAGMA table_info($#);" + + check db.getAllRows(sql qry % "Toy") == @[ + @[?0, ?"price", ?"FLOAT", ?1, ?nil, ?0], + @[?1, ?"id", ?"INTEGER", ?1, ?nil, ?1], + ] + + close pool + + test "Concurrent pool connections": + var + pool = newPool(2) + toy1 = newToy(123.45) + toy2 = newToy(456.78) + threads: array[2, Thread[float]] + sum: float + + withDb(pool): + db.createTables(toy1) + db.insert(toy1) + db.insert(toy2) + + proc getToy(price: float) {.thread.} = + {.cast(gcsafe).}: + var toy = newToy() + + withDb(pool): + db.select(toy, "price = ?", price) + + sum += toy.price + + createThread(threads[0], getToy, 123.45) + createThread(threads[1], getToy, 456.78) + + joinThreads(threads) + + check sum == toy1.price + toy2.price + + close pool + + test "Pool exhausted, raise exception": + var + pool = newPool(1, pepRaise) + toy1 = newToy(123.45) + toy2 = newToy(456.78) + threads: array[2, Thread[float]] + exceptionRaised: bool + + withDb(pool): + db.createTables(toy1) + db.insert(toy1) + db.insert(toy2) + + proc getToy(price: float) {.thread.} = + {.cast(gcsafe).}: + var toy = newToy() + + try: + withDb(pool): + db.select(toy, "price = ?", price) + while not exceptionRaised: + sleep 100 + except PoolExhaustedError: + exceptionRaised = true + + createThread(threads[0], getToy, 123.45) + createThread(threads[1], getToy, 456.78) + + joinThreads(threads) + + check exceptionRaised + + close pool + + test "Pool exhausted, extend and reset pool": + var + pool = newPool(1, pepExtend) + toy1 = newToy(123.45) + toy2 = newToy(456.78) + threads: array[2, Thread[float]] + maxActiveConnectionCount: Natural + + withDb(pool): + db.createTables(toy1) + db.insert(toy1) + db.insert(toy2) + + proc getToy(price: float) {.thread.} = + {.cast(gcsafe).}: + var toy = newToy() + + withDb(pool): + inc maxActiveConnectionCount + + db.select(toy, "price = ?", price) + + while maxActiveConnectionCount < 2: + sleep 100 + + createThread(threads[0], getToy, 123.45) + createThread(threads[1], getToy, 456.78) + + joinThreads(threads) + + check maxActiveConnectionCount == 2 + + reset pool + + check pool.size == pool.defaultSize + + close pool + diff --git a/tests/sqlite/tpool.nims b/tests/sqlite/tpool.nims new file mode 100644 index 00000000..84168203 --- /dev/null +++ b/tests/sqlite/tpool.nims @@ -0,0 +1,2 @@ +switch("threads", "on") + From 9c7171067e9bbb760a6dae0a1056e825651ab3e8 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 12:27:35 +0300 Subject: [PATCH 03/26] Add new test to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b4bb8e54..29b00f65 100755 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ tests/sqlite/tfkpragma_selfref tests/sqlite/tfkpragmanotint tests/sqlite/tfkpragmanotmodel tests/sqlite/tnull +tests/sqlite/tpool tests/sqlite/trelated tests/sqlite/trepeatfks tests/sqlite/trawselect From 20a6ef21c37efaa9ad7cf59df4c8cdf573e08908 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 12:27:52 +0300 Subject: [PATCH 04/26] Enable --mm:orc for tests. --- tests/config.nims | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config.nims b/tests/config.nims index 001b8a97..71c27c88 100755 --- a/tests/config.nims +++ b/tests/config.nims @@ -1,3 +1,4 @@ switch("path", "$projectDir/../src") switch("define", "normDebug") switch("deepcopy", "on") +switch("mm", "orc") From 2eb62a36d99a04e7833e7ea07103e64589a7d0a1 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 12:28:10 +0300 Subject: [PATCH 05/26] Minor formatting cleanup. --- tests/sqlite/tenv.nim | 1 + tests/sqlite/trows.nim | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/sqlite/tenv.nim b/tests/sqlite/tenv.nim index 48620c89..93d5b4c1 100644 --- a/tests/sqlite/tenv.nim +++ b/tests/sqlite/tenv.nim @@ -42,3 +42,4 @@ suite "DB config from environment variables": @[?0, ?"price", ?"FLOAT", ?1, ?nil, ?0], @[?1, ?"id", ?"INTEGER", ?1, ?nil, ?1], ] + diff --git a/tests/sqlite/trows.nim b/tests/sqlite/trows.nim index bb6dfd01..7647d546 100644 --- a/tests/sqlite/trows.nim +++ b/tests/sqlite/trows.nim @@ -19,7 +19,7 @@ suite "Row CRUD": removeFile dbFile test "Insert row": - var toy = newtoy(123.45) + var toy = newToy(123.45) dbConn.insert(toy) @@ -31,7 +31,7 @@ suite "Row CRUD": check rows[0] == @[?123.45, ?toy.id] test "Insert row twice": - var toy = newtoy(123.45) + var toy = newToy(123.45) dbConn.insert(toy) dbConn.insert(toy, force = true, conflictPolicy = cpReplace) @@ -44,31 +44,31 @@ suite "Row CRUD": check rows[^1] == @[?123.45, ?toy.id] test "Insert with forced id": - var toy = newtoy(137.45) + var toy = newToy(137.45) toy.id = 134 dbConn.insert(toy, force = true) check toy.id == 134 test "Insert row with forced id in non-incremental order": block: - var toy = newtoy(123.45) + var toy = newToy(123.45) toy.id = 3 dbConn.insert(toy, force=true) check toy.id == 3 block: - var toy = newtoy(123.45) + var toy = newToy(123.45) toy.id = 2 dbConn.insert(toy, force=true) check toy.id == 2 block: # Check no id conflict - var toy = newtoy(123.45) + var toy = newToy(123.45) dbConn.insert(toy) # SQLite ids starts from the highest ? check toy.id == 4 block: # Check no id conflict - var toy = newtoy(123.45) + var toy = newToy(123.45) dbConn.insert(toy) # SQLite ids starts from the highest ? check toy.id == 5 From 6cdf12fa72c5a1091c221a0f63d5a5d7bfaa2286 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 12:49:36 +0300 Subject: [PATCH 06/26] Add doc comments. --- src/norm/pool.nim | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/norm/pool.nim b/src/norm/pool.nim index 574278b4..dae8aa97 100644 --- a/src/norm/pool.nim +++ b/src/norm/pool.nim @@ -16,6 +16,13 @@ type func newPool*(defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool = + ##[ Create a connection pool of the given size. + + ``poolExhaustedPolicy`` defines how the pool reacts when a connection is requested but the pool has no connection available: + + - ``pepRaise`` (default) means throw ``PoolExhaustedError`` + - ``pepExtend`` means throw “add another connection to the pool.” + ]## result = Pool(defaultSize: defaultSize, conns: newSeq[DbConn](defaultSize), poolExhaustedPolicy: poolExhaustedPolicy) initLock(result.lock) @@ -30,6 +37,11 @@ func size*(pool: Pool): Natural = len(pool.conns) func pop*(pool: var Pool): DbConn = + ##[ Take a connection from the pool. + + If you're calling this manually, don't forget to `add <#add>`_ it back! + ]## + withLock(pool.lock): if pool.size > 0: result = pool.conns.pop() @@ -41,16 +53,25 @@ func pop*(pool: var Pool): DbConn = result = getDb() func add*(pool: var Pool, dbConn: DbConn) = + ##[ Add a connection to the pool. + + Use to return a borrowed connection to the pool. + ]## + withLock(pool.lock): pool.conns.add(dbConn) func reset*(pool: var Pool) = + ## Reset pool size to ``defaultSize`` by closing and removing extra connections. + withLock(pool.lock): while pool.size > pool.defaultSize: var conn = pool.conns.pop() close conn func close*(pool: var Pool) = + ## Close the pool by closing and removing all its connetions. + withLock(pool.lock): for conn in pool.conns.mitems: close conn @@ -58,6 +79,12 @@ func close*(pool: var Pool) = pool.conns.setLen(0) template withDb*(pool: var Pool, body: untyped): untyped = + ##[ Wrapper for DB operations. + + Takes a ``DbConn`` from a pool as ``db`` variable, + runs your code in a ``try`` block, and returns connection to the pool afterward. + ]## + block: let db {.inject.} = pool.pop() From 684782473e65b1441f21f9d03a5fa8c02ae707be Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 12:51:24 +0300 Subject: [PATCH 07/26] Define --mm:orc for the necessary test only. --- tests/config.nims | 1 - tests/sqlite/tpool.nims | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config.nims b/tests/config.nims index 71c27c88..001b8a97 100755 --- a/tests/config.nims +++ b/tests/config.nims @@ -1,4 +1,3 @@ switch("path", "$projectDir/../src") switch("define", "normDebug") switch("deepcopy", "on") -switch("mm", "orc") diff --git a/tests/sqlite/tpool.nims b/tests/sqlite/tpool.nims index 84168203..97174cea 100644 --- a/tests/sqlite/tpool.nims +++ b/tests/sqlite/tpool.nims @@ -1,2 +1,3 @@ switch("threads", "on") +switch("mm", "orc") From ff252e492e6ff4a8fa5bc6804c30ac70e1636695 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 13:05:05 +0300 Subject: [PATCH 08/26] Fix doc comments, add pool to API docs. --- src/norm.nim | 3 ++- src/norm/private/postgres/rowutils.nim | 18 +++++++++--------- src/norm/private/sqlite/rowutils.nim | 16 ++++++++-------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/norm.nim b/src/norm.nim index b8a146f7..9d7ae522 100755 --- a/src/norm.nim +++ b/src/norm.nim @@ -1,3 +1,4 @@ {.warning[UnusedImport]: off.} -import norm/[model, pragmas, sqlite, postgres] +import norm/[model, pragmas, sqlite, postgres, pool] + diff --git a/src/norm/private/postgres/rowutils.nim b/src/norm/private/postgres/rowutils.nim index 3faf9f30..be8283df 100644 --- a/src/norm/private/postgres/rowutils.nim +++ b/src/norm/private/postgres/rowutils.nim @@ -21,20 +21,20 @@ func isEmptyColumn*(row: Row, index: int): bool = ## This does the actual heavy lifting for parsing proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: static bool = false) = - ##[ Convert ``ndb.sqlite.Row`` instance into `ref object`_ instance, from a given position. + ##[ Convert ``ndb.sqlite.Row`` instance into `Model`_ instance, from a given position. - This is a helper proc to convert to `ref object`_ instances that have fields of the same type. + This is a helper proc to convert to `Model`_ instances that have fields of the same type. ]## for fld, dummyVal in T()[].fieldPairs: when isRefObject(typeof(dummyVal)): ## If we're dealing with a ``ref object`` field - if dot(obj, fld).toOptional().isSome: ## and it's either a ``some ref object`` or ``ref object`` + if dot(obj, fld).toOptional().isSome: ## and it's either a ``some Model`` or ``Model`` var subMod = dot(obj, fld).toOptional().get() ## then we try to populate it with the next ``row`` values. - if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty ``ref object``: - when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``ref object`` or an ``Option[ref object]`` at this point, + if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty ``Model``: + when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``Model`` or an ``Option[Model]`` at this point, when isRefObject(dummyVal): - dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[ref object]``, + dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[Model]``, inc pos subMod.fromRowPos(row, pos, skip = true) ## Then we skip all the ``row`` values that should've gone into this submodel. @@ -43,7 +43,7 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta inc pos ## subMod.fromRowPos(row, pos) ## we actually populate the submodel. - else: ## If the field is a ``none ref object``, + else: ## If the field is a ``none Model``, inc pos ## don't bother trying to populate it at all. else: @@ -56,9 +56,9 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta proc fromRow*[T: ref object](obj: var T, row: Row) = - ##[ Populate `ref object`_ instance from ``ndb.postgres.Row`` instance. + ##[ Populate `Model`_ instance from ``ndb.postgres.Row`` instance. - Nested `ref object`_ fields are populated from the same ``ndb.postgres.Row`` instance. + Nested `Model`_ fields are populated from the same ``ndb.postgres.Row`` instance. ]## var pos: Natural = 0 diff --git a/src/norm/private/sqlite/rowutils.nim b/src/norm/private/sqlite/rowutils.nim index 209f9695..bbecfc02 100644 --- a/src/norm/private/sqlite/rowutils.nim +++ b/src/norm/private/sqlite/rowutils.nim @@ -21,9 +21,9 @@ func isEmptyColumn*(row: Row, index: int): bool = ## This does the actual heavy lifting for parsing proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: static bool = false) = - ##[ Convert ``ndb.sqlite.Row`` instance into `ref object`_ instance, from a given position. + ##[ Convert ``ndb.sqlite.Row`` instance into `Model`_ instance, from a given position. - This is a helper proc to convert to `ref object`_ instances that have fields of the same type. + This is a helper proc to convert to `Model`_ instances that have fields of the same type. ]## for fld, dummyVal in T()[].fieldPairs: @@ -31,10 +31,10 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta if dot(obj, fld).toOptional().isSome: ## and it's either a ``some ref object`` or ``ref object`` var subMod = dot(obj, fld).toOptional().get() ## then we try to populate it with the next ``row`` values. - if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty ``ref object``: - when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``ref object`` or an ``Option[ref object]`` at this point, + if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty ``Model``: + when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``Model`` or an ``Option[Model]`` at this point, when isRefObject(dummyVal): - dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[ref object]``, + dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[Model]``, inc pos subMod.fromRowPos(row, pos, skip = true) ## Then we skip all the ``row`` values that should've gone into this submodel. @@ -43,7 +43,7 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta inc pos ## subMod.fromRowPos(row, pos) ## we actually populate the submodel. - else: ## If the field is a ``none ref object``, + else: ## If the field is a ``none Model``, inc pos ## don't bother trying to populate it at all. else: @@ -55,9 +55,9 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta inc pos proc fromRow*[T: ref object](obj: var T, row: Row) = - ##[ Populate `ref object`_ instance from ``ndb.sqlite.Row`` instance. + ##[ Populate `Model`_ instance from ``ndb.sqlite.Row`` instance. - Nested `ref object`_ fields are populated from the same ``ndb.sqlite.Row`` instance. + Nested `Model`_ fields are populated from the same ``ndb.sqlite.Row`` instance. ]## var pos: Natural = 0 From 41bdd850d6791e2689053761bd459979fd8bddc7 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 21:23:37 +0300 Subject: [PATCH 09/26] Minor formatting fixes. --- src/norm/postgres.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/norm/postgres.nim b/src/norm/postgres.nim index 82764f9a..52f035d4 100644 --- a/src/norm/postgres.nim +++ b/src/norm/postgres.nim @@ -1,4 +1,3 @@ - import std/[os, logging, strutils, sequtils, options, sugar, strformat, tables] when (NimMajor, NimMinor) <= (1, 6): @@ -38,7 +37,7 @@ const # Sugar to get DB config from environment variables -proc getDb*(): DbConn = +proc getDb*: DbConn = ## Create a ``DbConn`` from ``DB_HOST``, ``DB_USER``, ``DB_PASS``, and ``DB_NAME`` environment variables. open(getEnv(dbHostEnv), getEnv(dbUserEnv), getEnv(dbPassEnv), getEnv(dbNameEnv)) From ba029807db46a0598f6b09741356f15b57bfd591 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 21:23:50 +0300 Subject: [PATCH 10/26] Minor formatting fixes. --- src/norm/sqlite.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/norm/sqlite.nim b/src/norm/sqlite.nim index ef09c821..210f824a 100755 --- a/src/norm/sqlite.nim +++ b/src/norm/sqlite.nim @@ -32,7 +32,7 @@ const dbHostEnv* = "DB_HOST" # Sugar to get DB config from environment variables -proc getDb*(): DbConn = +proc getDb*: DbConn = ## Create a ``DbConn`` from ``DB_HOST`` environment variable. open(getEnv(dbHostEnv), "", "", "") From e94fb5ba78de00a2ec62b981c3cbc05efb0165f6 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 21:24:19 +0300 Subject: [PATCH 11/26] Remove redundant check. --- tests/postgres/ttables.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/postgres/ttables.nim b/tests/postgres/ttables.nim index 3f4bbc8a..101fa03a 100644 --- a/tests/postgres/ttables.nim +++ b/tests/postgres/ttables.nim @@ -67,8 +67,6 @@ suite "Table creation": dbConn.createTables(person) - check true - let qry = sql """SELECT column_name::text, data_type::text FROM information_schema.columns WHERE table_name = $1 From 20dee374222495a8c597450bf352b03299940abf Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 21:25:07 +0300 Subject: [PATCH 12/26] Update pool implenetation to support postgres. --- src/norm/pool.nim | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/norm/pool.nim b/src/norm/pool.nim index dae8aa97..512d1501 100644 --- a/src/norm/pool.nim +++ b/src/norm/pool.nim @@ -1,12 +1,13 @@ import std/locks -import sqlite +import sqlite, postgres type - Pool* = ref object + Conn = sqlite.DbConn | postgres.DbConn + Pool*[T: Conn] = ref object defaultSize: Natural - conns: seq[DbConn] + conns: seq[T] poolExhaustedPolicy: PoolExhaustedPolicy lock: Lock PoolExhaustedError* = object of CatchableError @@ -15,7 +16,7 @@ type pepExtend -func newPool*(defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool = +func newPool*[T: Conn](defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool[T] = ##[ Create a connection pool of the given size. ``poolExhaustedPolicy`` defines how the pool reacts when a connection is requested but the pool has no connection available: @@ -23,12 +24,17 @@ func newPool*(defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool = - ``pepRaise`` (default) means throw ``PoolExhaustedError`` - ``pepExtend`` means throw “add another connection to the pool.” ]## - result = Pool(defaultSize: defaultSize, conns: newSeq[DbConn](defaultSize), poolExhaustedPolicy: poolExhaustedPolicy) + + result = Pool[T](defaultSize: defaultSize, conns: newSeq[T](defaultSize), poolExhaustedPolicy: poolExhaustedPolicy) initLock(result.lock) for conn in result.conns.mitems: - conn = getDb() + conn = + when T is sqlite.DbConn: + sqlite.getDb() + elif T is postgres.DbConn: + postgres.getDb() func defaultSize*(pool: Pool): Natural = pool.defaultSize @@ -36,10 +42,10 @@ func defaultSize*(pool: Pool): Natural = func size*(pool: Pool): Natural = len(pool.conns) -func pop*(pool: var Pool): DbConn = +func pop*[T: Conn](pool: var Pool[T]): T = ##[ Take a connection from the pool. - If you're calling this manually, don't forget to `add <#add>`_ it back! + If you're calling this manually, don't forget to `add <#add,Pool,DbConn>`_ it back! ]## withLock(pool.lock): @@ -50,9 +56,13 @@ func pop*(pool: var Pool): DbConn = of pepRaise: raise newException(PoolExhaustedError, "Pool exhausted") of pepExtend: - result = getDb() + result = + when T is sqlite.DbConn: + sqlite.getDb() + elif T is postgres.DbConn: + postgres.getDb() -func add*(pool: var Pool, dbConn: DbConn) = +func add*[T: Conn](pool: var Pool, dbConn: T) = ##[ Add a connection to the pool. Use to return a borrowed connection to the pool. From 1e152cb9622176b56058f9592d4e7cd311814002 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 21:25:39 +0300 Subject: [PATCH 13/26] SQLite: Update pool tests to support the new implementation. --- tests/sqlite/tpool.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/sqlite/tpool.nim b/tests/sqlite/tpool.nim index 3c800326..242b7b9e 100644 --- a/tests/sqlite/tpool.nim +++ b/tests/sqlite/tpool.nim @@ -4,8 +4,10 @@ import norm/[sqlite, pool] import ../models + const dbFile = "test.db" + suite "Connection pool": setup: removeFile dbFile @@ -16,7 +18,7 @@ suite "Connection pool": removeFile dbFile test "Create and close pool": - var pool = newPool(1) + var pool = newPool[DbConn](1) check pool.defaultSize == 1 check pool.size == 1 @@ -26,7 +28,7 @@ suite "Connection pool": check pool.size == 0 test "Explicit pool connection": - var pool = newPool(1) + var pool = newPool[DbConn](1) let db = pool.pop() db.createTables(newToy()) @@ -42,7 +44,7 @@ suite "Connection pool": close pool test "Implicit pool connection": - var pool = newPool(1) + var pool = newPool[DbConn](1) withDb(pool): db.createTables(newToy()) @@ -58,7 +60,7 @@ suite "Connection pool": test "Concurrent pool connections": var - pool = newPool(2) + pool = newPool[DbConn](2) toy1 = newToy(123.45) toy2 = newToy(456.78) threads: array[2, Thread[float]] @@ -89,7 +91,7 @@ suite "Connection pool": test "Pool exhausted, raise exception": var - pool = newPool(1, pepRaise) + pool = newPool[DbConn](1, pepRaise) toy1 = newToy(123.45) toy2 = newToy(456.78) threads: array[2, Thread[float]] @@ -123,7 +125,7 @@ suite "Connection pool": test "Pool exhausted, extend and reset pool": var - pool = newPool(1, pepExtend) + pool = newPool[DbConn](1, pepExtend) toy1 = newToy(123.45) toy2 = newToy(456.78) threads: array[2, Thread[float]] From f106d68715866ffa57da45446cb397c5df25d3c3 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 21:26:00 +0300 Subject: [PATCH 14/26] Postgres: Add tests for pool. --- .gitignore | 1 + tests/postgres/tpool.nim | 187 ++++++++++++++++++++++++++++++++++++++ tests/postgres/tpool.nims | 3 + 3 files changed, 191 insertions(+) create mode 100644 tests/postgres/tpool.nim create mode 100644 tests/postgres/tpool.nims diff --git a/.gitignore b/.gitignore index 29b00f65..4c4809c6 100755 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ tests/postgres/tfkpragma_selfref tests/postgres/tfkpragmanotint tests/postgres/tfkpragmanotmodel tests/postgres/tnull +tests/postgres/tpool tests/postgres/trelated tests/postgres/trepeatfks tests/postgres/trawselect diff --git a/tests/postgres/tpool.nim b/tests/postgres/tpool.nim new file mode 100644 index 00000000..ddb44482 --- /dev/null +++ b/tests/postgres/tpool.nim @@ -0,0 +1,187 @@ +import std/[unittest, os, strutils] + +import norm/[postgres, pool] + +import ../models + + +const + dbHost = "postgres" + dbUser = "postgres" + dbPassword = "postgres" + dbDatabase = "postgres" + + +suite "Connection pool": + proc resetDb = + let dbConn = open(dbHost, dbUser, dbPassword, "template1") + dbConn.exec(sql "DROP DATABASE IF EXISTS $#" % dbDatabase) + dbConn.exec(sql "CREATE DATABASE $#" % dbDatabase) + close dbConn + + setup: + resetDb() + + putEnv(dbHostEnv, dbHost) + putEnv(dbUserEnv, dbUser) + putEnv(dbPassEnv, dbPassword) + putEnv(dbNameEnv, dbDatabase) + + teardown: + delEnv(dbHostEnv) + delEnv(dbUserEnv) + delEnv(dbPassEnv) + delEnv(dbNameEnv) + + resetDb() + + test "Create and close pool": + var pool = newPool[DbConn](1) + + check pool.defaultSize == 1 + check pool.size == 1 + + close pool + + check pool.size == 0 + + test "Explicit pool connection": + var pool = newPool[DbConn](1) + let db = pool.pop() + + db.createTables(newToy()) + + let qry = sql """SELECT column_name::text, data_type::text + FROM information_schema.columns + WHERE table_name = $1 + ORDER BY column_name""" + + check db.getAllRows(qry, "Toy") == @[ + @[?"id", ?"bigint"], + @[?"price", ?"double precision"] + ] + + pool.add(db) + close pool + + test "Implicit pool connection": + var pool = newPool[DbConn](1) + + withDb(pool): + db.createTables(newToy()) + + let qry = sql """SELECT column_name::text, data_type::text + FROM information_schema.columns + WHERE table_name = $1 + ORDER BY column_name""" + + check db.getAllRows(qry, "Toy") == @[ + @[?"id", ?"bigint"], + @[?"price", ?"double precision"] + ] + + close pool + + test "Concurrent pool connections": + var + pool = newPool[DbConn](2) + toy1 = newToy(123.45) + toy2 = newToy(456.78) + threads: array[2, Thread[float]] + sum: float + + withDb(pool): + db.createTables(toy1) + db.insert(toy1) + db.insert(toy2) + + proc getToy(price: float) {.thread.} = + {.cast(gcsafe).}: + var toy = newToy() + + withDb(pool): + db.select(toy, "price = $1", price) + + sum += toy.price + + createThread(threads[0], getToy, 123.45) + createThread(threads[1], getToy, 456.78) + + joinThreads(threads) + + check sum == toy1.price + toy2.price + + close pool + + test "Pool exhausted, raise exception": + var + pool = newPool[DbConn](1, pepRaise) + toy1 = newToy(123.45) + toy2 = newToy(456.78) + threads: array[2, Thread[float]] + exceptionRaised: bool + + withDb(pool): + db.createTables(toy1) + db.insert(toy1) + db.insert(toy2) + + proc getToy(price: float) {.thread.} = + {.cast(gcsafe).}: + var toy = newToy() + + try: + withDb(pool): + db.select(toy, "price = $1", price) + while not exceptionRaised: + sleep 100 + except PoolExhaustedError: + exceptionRaised = true + + createThread(threads[0], getToy, 123.45) + createThread(threads[1], getToy, 456.78) + + joinThreads(threads) + + check exceptionRaised + + close pool + + test "Pool exhausted, extend and reset pool": + var + pool = newPool[DbConn](1, pepExtend) + toy1 = newToy(123.45) + toy2 = newToy(456.78) + threads: array[2, Thread[float]] + maxActiveConnectionCount: Natural + + withDb(pool): + db.createTables(toy1) + db.insert(toy1) + db.insert(toy2) + + proc getToy(price: float) {.thread.} = + {.cast(gcsafe).}: + var toy = newToy() + + withDb(pool): + inc maxActiveConnectionCount + + db.select(toy, "price = $1", price) + + while maxActiveConnectionCount < 2: + sleep 100 + + createThread(threads[0], getToy, 123.45) + createThread(threads[1], getToy, 456.78) + + joinThreads(threads) + + check maxActiveConnectionCount == 2 + + reset pool + + check pool.size == pool.defaultSize + + close pool + diff --git a/tests/postgres/tpool.nims b/tests/postgres/tpool.nims new file mode 100644 index 00000000..97174cea --- /dev/null +++ b/tests/postgres/tpool.nims @@ -0,0 +1,3 @@ +switch("threads", "on") +switch("mm", "orc") + From 45d15ae5087681e1d7b91a82b8cffaed449db148 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 21:32:29 +0300 Subject: [PATCH 15/26] Add blank line. --- src/norm/sqlite.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/src/norm/sqlite.nim b/src/norm/sqlite.nim index 210f824a..51bf388e 100755 --- a/src/norm/sqlite.nim +++ b/src/norm/sqlite.nim @@ -27,6 +27,7 @@ type ]## NotFoundError* = object of KeyError + const dbHostEnv* = "DB_HOST" From d1447073252e795d0a726194d1549af61358b61d Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 22:16:10 +0300 Subject: [PATCH 16/26] Add blank lines. --- book/tutorial/rawSelects.nim | 3 ++- book/tutorial/rows.nim | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/book/tutorial/rawSelects.nim b/book/tutorial/rawSelects.nim index ea640332..1f80279a 100644 --- a/book/tutorial/rawSelects.nim +++ b/book/tutorial/rawSelects.nim @@ -5,6 +5,7 @@ nbInit(theme = useNimibook) nbText: """ # Raw SQL SELECT interactions + Sometimes SQL abilities are needed that norm can not represent well. For such cases, norm provides a way to execute raw SQL SELECT queries and parse the received data into a user provided ``ref object`` type. @@ -52,4 +53,4 @@ nbCode: echo countResult.count -nbSave \ No newline at end of file +nbSave diff --git a/book/tutorial/rows.nim b/book/tutorial/rows.nim index 31815fb8..f45b51df 100644 --- a/book/tutorial/rows.nim +++ b/book/tutorial/rows.nim @@ -62,7 +62,9 @@ nbCode: nbText: &""" ## Select Row + ### Select in general + To select a rows with Norm, you instantiate a model that serves as a container for the selected data and call `select`. One curious thing about `select` is that its result depends not only on the condition you pass but also on the container. If the container has `Model` fields that are not `None`, Norm will select the related rows in a single `JOIN` query giving you a fully populated model object. However, if the container has a `none Model` field, it is just ignored. @@ -137,6 +139,7 @@ nbCode: nbText: """ ### Selecting Many-To-One/One-To-Many relationships + Imagine you had a Many-To-One relationship between two models, like we have with `Customer` being the many-model and `User` being the one-model, where one user can have many customers. If you have a user and wanted to query all of their customers, you couldn't do so by just making a query for the user, as that model doesn't have a "seq[Customer]" field that norm could resolve. @@ -173,7 +176,9 @@ In the first approach, if Customer doesn't have a field called "user" or if that In the second approach, if Customer doesn't have any field of type "User" or any other model-type that points to the same table as "User", it will also not compile while throwing a helpful error message. + ### Selecting Many-To-Many relationships + Imagine if you had a Many-To-Many relationship between two models (e.g. Users and Groups) that is recorded on an "join-model" (e.g. UserGroup), where one user can be in many groups and a group can have many users. If you have a user and want to query all of its groups, you can do so via the general select statement mechanism. From b3a5217c054368bfe759fe8b4f981bb7417d3139 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 22:16:29 +0300 Subject: [PATCH 17/26] Book: Add chapter on connection pooling. --- book/pool.nim | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ nbook.nim | 1 + 2 files changed, 63 insertions(+) create mode 100644 book/pool.nim diff --git a/book/pool.nim b/book/pool.nim new file mode 100644 index 00000000..f74c4f33 --- /dev/null +++ b/book/pool.nim @@ -0,0 +1,62 @@ +import nimib, nimibook + + +nbInit(theme = useNimibook) + +nbText: """ +# Connection Pool + +Connection pooling is a technique that involves precreating and reusing a number of ever-open DB connections instead of opening connections on demand. Since opening and closing connections takes more time than passing open connections around, this technique is used to improve performance of web application under high load. + +Norm offers a simple thread-safe connection pool implementation. It can be used with both Postgres and SQLite, and you even can create multiple pools if you need to. + +**Important** Connection pooling requires ``--mm:orc``. + +To use connection pool: +1. Create a ``Pool`` instance by calling ``newPool[DbConn]()``. +2. Wrap your DB calls in a ``withDb()`` block. + +``newPool`` creates a pool of size ```` with connections of type ``DbConn`` (either ``sqlite.DbConn`` or ``postgres.DbConn``). The params for the connections are taken from the environment, similar to how ``withDb`` works (see `Configuration from Environment `_). +""" + +nbCode: + import norm/[model, sqlite, pool] + + + type + Product = ref object of Model + name: string + price: float + + proc newProduct(): Product = + Product(name: "", price: 0.0) + + putEnv("DB_HOST", ":memory:") + + var connPool = newPool[DbConn](10) + + withDb(connPool): + db.exec sql"PRAGMA foreign_keys = ON" + db.createTables(newProduct()) + +nbText: """ +## Pool Exhausted Policy + +If the app requests more connections from the pool than it can give, we say the pool is exhausted. + +There are two ways a pool can react to that: +1. Raise a ``PoolExhaustedError``. This is the default policy. +2. Open an additional connection and extend the pool size. + +The policy is set during the pool creation by setting ``poolExhaustedPolicy`` param to either ``pepRaise`` or ``pepExtend``. + + +## Manual Pool Manipulation + +You can borrow connections from the pool manually by calling ``.pop()`` proc. + +**Important** If you choose to get connections from the pool manually, you must care about putting the borrowed connections back byb calling ``.add()``. +""" + +nbSave + diff --git a/nbook.nim b/nbook.nim index 653923b9..518d27b0 100644 --- a/nbook.nim +++ b/nbook.nim @@ -12,6 +12,7 @@ var book = initBookWithToc: entry("Fancy Syntax", "fancy.nim") entry("Transactions", "transactions.nim") entry("Configuration from Environment", "config.nim") + entry("Connection Pool", "pool.nim") entry("Manual Foreign Key Handling", "fk.nim") entry("Custom Datatypes", "customDatatypes.nim") entry("Debugging SQL", "debug.nim") From bf3e13b14329d81c65a1c742b6aa899514ebda33 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 22:54:48 +0300 Subject: [PATCH 18/26] Book: Improve chapter on pooling. --- book/pool.nim | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/book/pool.nim b/book/pool.nim index f74c4f33..a2f84c78 100644 --- a/book/pool.nim +++ b/book/pool.nim @@ -10,13 +10,13 @@ Connection pooling is a technique that involves precreating and reusing a number Norm offers a simple thread-safe connection pool implementation. It can be used with both Postgres and SQLite, and you even can create multiple pools if you need to. -**Important** Connection pooling requires ``--mm:orc``. +**Important** Connection pooling requires `--mm:orc`. To use connection pool: -1. Create a ``Pool`` instance by calling ``newPool[DbConn]()``. -2. Wrap your DB calls in a ``withDb()`` block. +1. create a [`Pool`](/apidocs/norm/pool.html#Pool) instance by calling [`newPool`](/apidocs/norm/pool.html#newPool,Positive) +2. wrap your DB calls in a [`withDb`](/apidocs/norm/pool.html#withDb.t,Pool,untyped) -``newPool`` creates a pool of size ```` with connections of type ``DbConn`` (either ``sqlite.DbConn`` or ``postgres.DbConn``). The params for the connections are taken from the environment, similar to how ``withDb`` works (see `Configuration from Environment `_). +`newPool` creates a pool of a given size with connections of type `DbConn` (either from [`sqlite`](/apidocs/norm/sqlite.html) or [`postgres`](/apidocs/norm/postgres.html)). The params for the connections are taken from the environment, similar to how `withDb` works (see [Configuration from Environment](/config.html)). """ nbCode: @@ -45,17 +45,19 @@ nbText: """ If the app requests more connections from the pool than it can give, we say the pool is exhausted. There are two ways a pool can react to that: -1. Raise a ``PoolExhaustedError``. This is the default policy. -2. Open an additional connection and extend the pool size. +1. raise a [`PoolExhaustedError`](/apidocs/norm/pool.html#PoolExhaustedError); this is the default policy +2. open an additional connection and extend the pool size -The policy is set during the pool creation by setting ``poolExhaustedPolicy`` param to either ``pepRaise`` or ``pepExtend``. +The policy is set during the pool creation by setting `poolExhaustedPolicy` param to either `pepRaise` or `pepExtend`. + +To reset the pool back to its default size after it has been extended, call [`reset`](/apidocs/norm/pool.html#reset,Pool) proc on it. ## Manual Pool Manipulation -You can borrow connections from the pool manually by calling ``.pop()`` proc. +You can borrow connections from the pool manually by calling [`pop`](/apidocs/norm/pool.html#add%2CPool%2CT) proc. -**Important** If you choose to get connections from the pool manually, you must care about putting the borrowed connections back byb calling ``.add()``. +**Important** If you choose to get connections from the pool manually, you must care about putting the borrowed connections back byb calling [`add`](/apidocs/norm/pool.html#add,Pool,T). """ nbSave From 7cf32714dee8e977a0646823540f2d17fb21204d Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sat, 15 Oct 2022 23:05:41 +0300 Subject: [PATCH 19/26] Update changelog. --- changelog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changelog.md b/changelog.md index 116547e9..766dbd9e 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,11 @@ - [d]—docs improvement +## 2.5.3 (WIP) + +- [+] App connection pool (see [#50](https://github.com/moigagoo/norm/issues/50)). + + ## 2.5.2 (September 14, 2022) - [+] Added `rawSelect` proc, which allows you execute raw SQL and have the output be parsed into a custom object-type. @@ -15,6 +20,7 @@ - [r] Logging: refactored `log` module to not trigger warnings when `normDebug` is not defined. - [r] Slightly changed how objects are being parsed, leading to a small performance increase. + ## 2.5.1 (July 20, 2022) - [+] Added `uniqueGroup` pragma to provide UNIQUE constraint on multiple columns (see [#136](https://github.com/moigagoo/norm/issues/136)). - [+] Add `readOnly` alias for `ro` pragma (see [#128](https://github.com/moigagoo/norm/issues/128)). From 60a2b22f0e8fdad2b307b6ce4ada03edaf9a624f Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sun, 16 Oct 2022 11:05:35 +0300 Subject: [PATCH 20/26] Rename Conn to DbConn for consistency. --- src/norm/pool.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/norm/pool.nim b/src/norm/pool.nim index 512d1501..ccd46167 100644 --- a/src/norm/pool.nim +++ b/src/norm/pool.nim @@ -4,8 +4,8 @@ import sqlite, postgres type - Conn = sqlite.DbConn | postgres.DbConn - Pool*[T: Conn] = ref object + DbConn = sqlite.DbConn | postgres.DbConn + Pool*[T: DbConn] = ref object defaultSize: Natural conns: seq[T] poolExhaustedPolicy: PoolExhaustedPolicy @@ -16,7 +16,7 @@ type pepExtend -func newPool*[T: Conn](defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool[T] = +func newPool*[T: DbConn](defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool[T] = ##[ Create a connection pool of the given size. ``poolExhaustedPolicy`` defines how the pool reacts when a connection is requested but the pool has no connection available: @@ -42,7 +42,7 @@ func defaultSize*(pool: Pool): Natural = func size*(pool: Pool): Natural = len(pool.conns) -func pop*[T: Conn](pool: var Pool[T]): T = +func pop*[T: DbConn](pool: var Pool[T]): T = ##[ Take a connection from the pool. If you're calling this manually, don't forget to `add <#add,Pool,DbConn>`_ it back! @@ -62,7 +62,7 @@ func pop*[T: Conn](pool: var Pool[T]): T = elif T is postgres.DbConn: postgres.getDb() -func add*[T: Conn](pool: var Pool, dbConn: T) = +func add*[T: DbConn](pool: var Pool, dbConn: T) = ##[ Add a connection to the pool. Use to return a borrowed connection to the pool. From c4798bea674251fd88151d7d056ee55a62dfe54f Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sun, 16 Oct 2022 13:57:24 +0300 Subject: [PATCH 21/26] rowutils: Fix docstrings. --- src/norm/private/postgres/rowutils.nim | 18 +++++++++--------- src/norm/private/sqlite/rowutils.nim | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/norm/private/postgres/rowutils.nim b/src/norm/private/postgres/rowutils.nim index be8283df..343c0b4f 100644 --- a/src/norm/private/postgres/rowutils.nim +++ b/src/norm/private/postgres/rowutils.nim @@ -21,20 +21,20 @@ func isEmptyColumn*(row: Row, index: int): bool = ## This does the actual heavy lifting for parsing proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: static bool = false) = - ##[ Convert ``ndb.sqlite.Row`` instance into `Model`_ instance, from a given position. + ##[ Convert ``ndb.sqlite.Row`` instance into ``ref object`` instance, from a given position. - This is a helper proc to convert to `Model`_ instances that have fields of the same type. + This is a helper proc to convert to ``ref object`` instances that have fields of the same type. ]## for fld, dummyVal in T()[].fieldPairs: when isRefObject(typeof(dummyVal)): ## If we're dealing with a ``ref object`` field - if dot(obj, fld).toOptional().isSome: ## and it's either a ``some Model`` or ``Model`` + if dot(obj, fld).toOptional().isSome: ## and it's either a ``some ref object`` or ``ref object`` var subMod = dot(obj, fld).toOptional().get() ## then we try to populate it with the next ``row`` values. - if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty ``Model``: - when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``Model`` or an ``Option[Model]`` at this point, + if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty instance: + when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``ref object`` or an ``Option[ref object]`` at this point, when isRefObject(dummyVal): - dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[Model]``, + dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[ref object]``, inc pos subMod.fromRowPos(row, pos, skip = true) ## Then we skip all the ``row`` values that should've gone into this submodel. @@ -43,7 +43,7 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta inc pos ## subMod.fromRowPos(row, pos) ## we actually populate the submodel. - else: ## If the field is a ``none Model``, + else: ## If the field is a ``none ref object``, inc pos ## don't bother trying to populate it at all. else: @@ -56,9 +56,9 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta proc fromRow*[T: ref object](obj: var T, row: Row) = - ##[ Populate `Model`_ instance from ``ndb.postgres.Row`` instance. + ##[ Populate ``ref object`` instance from ``ndb.postgres.Row`` instance. - Nested `Model`_ fields are populated from the same ``ndb.postgres.Row`` instance. + Nested ``ref object`` fields are populated from the same ``ndb.postgres.Row`` instance. ]## var pos: Natural = 0 diff --git a/src/norm/private/sqlite/rowutils.nim b/src/norm/private/sqlite/rowutils.nim index bbecfc02..c9bcc64d 100644 --- a/src/norm/private/sqlite/rowutils.nim +++ b/src/norm/private/sqlite/rowutils.nim @@ -21,9 +21,9 @@ func isEmptyColumn*(row: Row, index: int): bool = ## This does the actual heavy lifting for parsing proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: static bool = false) = - ##[ Convert ``ndb.sqlite.Row`` instance into `Model`_ instance, from a given position. + ##[ Convert ``ndb.sqlite.Row`` instance into a ``ref object`` instance, from a given position. - This is a helper proc to convert to `Model`_ instances that have fields of the same type. + This is a helper proc to convert to ``ref object`` instances that have fields of the same type. ]## for fld, dummyVal in T()[].fieldPairs: @@ -31,10 +31,10 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta if dot(obj, fld).toOptional().isSome: ## and it's either a ``some ref object`` or ``ref object`` var subMod = dot(obj, fld).toOptional().get() ## then we try to populate it with the next ``row`` values. - if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty ``Model``: - when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``Model`` or an ``Option[Model]`` at this point, + if row.isEmptyColumn(pos): ## If we have a ``NULL`` at this point, we return an empty instance: + when typeof(dummyVal) is Option: ## ``val`` is guaranteed to be either ``ref object `` or an ``Option[ref object]`` at this point, when isRefObject(dummyVal): - dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[Model]``, + dot(obj, fld) = none typeof(subMod) ## and the fact that we got a ``NULL`` tells us it's an ``Option[ref object]``, inc pos subMod.fromRowPos(row, pos, skip = true) ## Then we skip all the ``row`` values that should've gone into this submodel. @@ -55,9 +55,9 @@ proc fromRowPos[T: ref object](obj: var T, row: Row, pos: var Natural, skip: sta inc pos proc fromRow*[T: ref object](obj: var T, row: Row) = - ##[ Populate `Model`_ instance from ``ndb.sqlite.Row`` instance. + ##[ Populate ``ref object`` instance from ``ndb.sqlite.Row`` instance. - Nested `Model`_ fields are populated from the same ``ndb.sqlite.Row`` instance. + Nested ``ref object`` fields are populated from the same ``ndb.sqlite.Row`` instance. ]## var pos: Natural = 0 From d372fa9bf1db6e1c1a9003da87ab683b4f659b0c Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sun, 16 Oct 2022 14:12:23 +0300 Subject: [PATCH 22/26] Book: Pool: Add code samples and prose on closing the pool. --- book/pool.nim | 21 +++++++++++++++++++++ norm.nimble | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/book/pool.nim b/book/pool.nim index a2f84c78..a3aef1d7 100644 --- a/book/pool.nim +++ b/book/pool.nim @@ -60,5 +60,26 @@ You can borrow connections from the pool manually by calling [`pop`](/apidocs/no **Important** If you choose to get connections from the pool manually, you must care about putting the borrowed connections back byb calling [`add`](/apidocs/norm/pool.html#add,Pool,T). """ +nbCode: + let dbConn = connPool.pop() + + var product = newProduct() + + product.name = "Table" + product.price = 123.45 + + dbConn.insert(product) + + connPool.add(dbConn) + +nbText: """ +## Closing the Pool + +When you no longer need the pool, for example, when your app exits or crashes, to avoid leaving hanging connections, close the pool by calling [`close`](/apidocs/norm/pool.html#close,Pool). This proc closes all connections in the pool and sets its size to 0. +""" + +nbCode: + close connPool + nbSave diff --git a/norm.nimble b/norm.nimble index bf96448a..092f85c8 100755 --- a/norm.nimble +++ b/norm.nimble @@ -17,7 +17,7 @@ task test, "Run tests": task book, "Generate book": rmDir "docs" - exec "nimble install -y nimib@#head nimibook@#head" +# exec "nimble install -y nimib@#head nimibook@#head" exec "nim r -d:release nbook.nim update" exec "nim r -d:release nbook.nim build" cpFile("CNAME", "docs/CNAME") From bec892f908a1df1d4054bc84a80a2f7d28c03d84 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sun, 16 Oct 2022 15:22:28 +0300 Subject: [PATCH 23/26] Changelog: Set next version to 2.6.0 --- changelog.md | 2 +- src/norm/private/log.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 766dbd9e..d86ef338 100644 --- a/changelog.md +++ b/changelog.md @@ -8,7 +8,7 @@ - [d]—docs improvement -## 2.5.3 (WIP) +## 2.6.0 (WIP) - [+] App connection pool (see [#50](https://github.com/moigagoo/norm/issues/50)). diff --git a/src/norm/private/log.nim b/src/norm/private/log.nim index 57ea19e1..b26f93d9 100644 --- a/src/norm/private/log.nim +++ b/src/norm/private/log.nim @@ -2,7 +2,7 @@ type LoggingError* = object of CatchableError -when defined(normdebug): +when defined(normDebug): import std/[logging, strutils] proc log*(msg: string) {.raises: LoggingError.} = From 912e948f3c0e4a4a4ac5ea455dbefb09c5b57563 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sun, 16 Oct 2022 15:40:31 +0300 Subject: [PATCH 24/26] Fix book task. --- norm.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/norm.nimble b/norm.nimble index 092f85c8..bf96448a 100755 --- a/norm.nimble +++ b/norm.nimble @@ -17,7 +17,7 @@ task test, "Run tests": task book, "Generate book": rmDir "docs" -# exec "nimble install -y nimib@#head nimibook@#head" + exec "nimble install -y nimib@#head nimibook@#head" exec "nim r -d:release nbook.nim update" exec "nim r -d:release nbook.nim build" cpFile("CNAME", "docs/CNAME") From 70f93f9eb0a378218926b966ad7d72c661dddc90 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sun, 16 Oct 2022 15:40:59 +0300 Subject: [PATCH 25/26] Allow arbitrary getDb function to be passed to newPool. --- src/norm/pool.nim | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/norm/pool.nim b/src/norm/pool.nim index ccd46167..bb375169 100644 --- a/src/norm/pool.nim +++ b/src/norm/pool.nim @@ -9,6 +9,7 @@ type defaultSize: Natural conns: seq[T] poolExhaustedPolicy: PoolExhaustedPolicy + getDbProc: proc: T lock: Lock PoolExhaustedError* = object of CatchableError PoolExhaustedPolicy* = enum @@ -16,7 +17,7 @@ type pepExtend -func newPool*[T: DbConn](defaultSize: Positive, poolExhaustedPolicy = pepRaise): Pool[T] = +proc newPool*[T: DbConn](defaultSize: Positive, poolExhaustedPolicy = pepRaise, getDbProc: proc: T = getDb): Pool[T] = ##[ Create a connection pool of the given size. ``poolExhaustedPolicy`` defines how the pool reacts when a connection is requested but the pool has no connection available: @@ -25,16 +26,12 @@ func newPool*[T: DbConn](defaultSize: Positive, poolExhaustedPolicy = pepRaise): - ``pepExtend`` means throw “add another connection to the pool.” ]## - result = Pool[T](defaultSize: defaultSize, conns: newSeq[T](defaultSize), poolExhaustedPolicy: poolExhaustedPolicy) + result = Pool[T](defaultSize: defaultSize, conns: newSeq[T](defaultSize), poolExhaustedPolicy: poolExhaustedPolicy, getDbProc: getDbProc) initLock(result.lock) for conn in result.conns.mitems: - conn = - when T is sqlite.DbConn: - sqlite.getDb() - elif T is postgres.DbConn: - postgres.getDb() + conn = result.getDbProc() func defaultSize*(pool: Pool): Natural = pool.defaultSize @@ -42,7 +39,7 @@ func defaultSize*(pool: Pool): Natural = func size*(pool: Pool): Natural = len(pool.conns) -func pop*[T: DbConn](pool: var Pool[T]): T = +proc pop*[T: DbConn](pool: var Pool[T]): T = ##[ Take a connection from the pool. If you're calling this manually, don't forget to `add <#add,Pool,DbConn>`_ it back! @@ -56,11 +53,7 @@ func pop*[T: DbConn](pool: var Pool[T]): T = of pepRaise: raise newException(PoolExhaustedError, "Pool exhausted") of pepExtend: - result = - when T is sqlite.DbConn: - sqlite.getDb() - elif T is postgres.DbConn: - postgres.getDb() + result = pool.getDbProc() func add*[T: DbConn](pool: var Pool, dbConn: T) = ##[ Add a connection to the pool. From 75c96103fbf6a1c0175c2e10b63d8397dbfb2fcd Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Sun, 16 Oct 2022 17:38:19 +0300 Subject: [PATCH 26/26] Add tests for pooling with a custom connection provider. --- book/pool.nim | 21 +++++++++++++++++++++ src/norm/pool.nim | 6 +++--- tests/postgres/tpool.nim | 25 +++++++++++++++++++++++-- tests/sqlite/tpool.nim | 19 +++++++++++++++++-- 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/book/pool.nim b/book/pool.nim index a3aef1d7..7851b008 100644 --- a/book/pool.nim +++ b/book/pool.nim @@ -81,5 +81,26 @@ When you no longer need the pool, for example, when your app exits or crashes, t nbCode: close connPool +nbText: """ +## Custom Connection Source + +By default, new connections are added to the pool by calling ``getDB``, which takes the DB params from the environment. + +But you can override that. For example, to get one pool connected to one DB and another one connected to another one. + +To do that, pass a function that returns ``DbConn`` to the Pool constructor: +""" + +nbCode: + func myDb: DbConn = open("mydb.db", "", "", "") + + var anotherPool = newPool[DbConn](10, myDb) + + assert anotherPool.size == 10 + assert fileExists("mydb.db") + + close anotherPool + removeFile("mydb.db") + nbSave diff --git a/src/norm/pool.nim b/src/norm/pool.nim index bb375169..35871f74 100644 --- a/src/norm/pool.nim +++ b/src/norm/pool.nim @@ -8,8 +8,8 @@ type Pool*[T: DbConn] = ref object defaultSize: Natural conns: seq[T] - poolExhaustedPolicy: PoolExhaustedPolicy getDbProc: proc: T + poolExhaustedPolicy: PoolExhaustedPolicy lock: Lock PoolExhaustedError* = object of CatchableError PoolExhaustedPolicy* = enum @@ -17,7 +17,7 @@ type pepExtend -proc newPool*[T: DbConn](defaultSize: Positive, poolExhaustedPolicy = pepRaise, getDbProc: proc: T = getDb): Pool[T] = +proc newPool*[T: DbConn](defaultSize: Positive, getDbProc: proc: T = getDb, poolExhaustedPolicy = pepRaise): Pool[T] = ##[ Create a connection pool of the given size. ``poolExhaustedPolicy`` defines how the pool reacts when a connection is requested but the pool has no connection available: @@ -26,7 +26,7 @@ proc newPool*[T: DbConn](defaultSize: Positive, poolExhaustedPolicy = pepRaise, - ``pepExtend`` means throw “add another connection to the pool.” ]## - result = Pool[T](defaultSize: defaultSize, conns: newSeq[T](defaultSize), poolExhaustedPolicy: poolExhaustedPolicy, getDbProc: getDbProc) + result = Pool[T](defaultSize: defaultSize, conns: newSeq[T](defaultSize), getDbProc: getDbProc, poolExhaustedPolicy: poolExhaustedPolicy) initLock(result.lock) diff --git a/tests/postgres/tpool.nim b/tests/postgres/tpool.nim index ddb44482..f65b988b 100644 --- a/tests/postgres/tpool.nim +++ b/tests/postgres/tpool.nim @@ -45,6 +45,27 @@ suite "Connection pool": check pool.size == 0 + test "Create and close pool with custom connection provider": + var dbConn = open(dbHost, dbUser, dbPassword, "template1") + dbConn.exec(sql "DROP DATABASE IF EXISTS $#" % "postgres2") + dbConn.exec(sql "CREATE DATABASE $#" % "postgres2") + close dbConn + + func myDb: DbConn = open(dbHost, dbUser, dbPassword, "postgres2") + + var pool = newPool[DbConn](1, myDb) + + check pool.defaultSize == 1 + check pool.size == 1 + + close pool + + check pool.size == 0 + + dbConn = open(dbHost, dbUser, dbPassword, "template1") + dbConn.exec(sql "DROP DATABASE IF EXISTS $#" % "postgres2") + close dbConn + test "Explicit pool connection": var pool = newPool[DbConn](1) let db = pool.pop() @@ -115,7 +136,7 @@ suite "Connection pool": test "Pool exhausted, raise exception": var - pool = newPool[DbConn](1, pepRaise) + pool = newPool[DbConn](1, poolExhaustedPolicy = pepRaise) toy1 = newToy(123.45) toy2 = newToy(456.78) threads: array[2, Thread[float]] @@ -149,7 +170,7 @@ suite "Connection pool": test "Pool exhausted, extend and reset pool": var - pool = newPool[DbConn](1, pepExtend) + pool = newPool[DbConn](1, poolExhaustedPolicy = pepExtend) toy1 = newToy(123.45) toy2 = newToy(456.78) threads: array[2, Thread[float]] diff --git a/tests/sqlite/tpool.nim b/tests/sqlite/tpool.nim index 242b7b9e..5f295518 100644 --- a/tests/sqlite/tpool.nim +++ b/tests/sqlite/tpool.nim @@ -27,6 +27,21 @@ suite "Connection pool": check pool.size == 0 + test "Create and close pool with custom connection provider": + func myDb: DbConn = open("test2.db", "", "", "") + + var pool = newPool[DbConn](1, myDb) + + check pool.defaultSize == 1 + check pool.size == 1 + check fileExists("test2.db") + + close pool + + check pool.size == 0 + + removeFile("test2.db") + test "Explicit pool connection": var pool = newPool[DbConn](1) let db = pool.pop() @@ -91,7 +106,7 @@ suite "Connection pool": test "Pool exhausted, raise exception": var - pool = newPool[DbConn](1, pepRaise) + pool = newPool[DbConn](1, poolExhaustedPolicy = pepRaise) toy1 = newToy(123.45) toy2 = newToy(456.78) threads: array[2, Thread[float]] @@ -125,7 +140,7 @@ suite "Connection pool": test "Pool exhausted, extend and reset pool": var - pool = newPool[DbConn](1, pepExtend) + pool = newPool[DbConn](1, poolExhaustedPolicy = pepExtend) toy1 = newToy(123.45) toy2 = newToy(456.78) threads: array[2, Thread[float]]