From c9f7073409eea590fc380e26c4c24fdff6b69868 Mon Sep 17 00:00:00 2001 From: CebullaD Date: Thu, 13 Mar 2025 18:00:23 +0100 Subject: [PATCH 1/5] Introduce parameters to control the connection pool for engine creation --- qwc_services_core/database.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/qwc_services_core/database.py b/qwc_services_core/database.py index c6241b0..e53e89c 100644 --- a/qwc_services_core/database.py +++ b/qwc_services_core/database.py @@ -1,7 +1,7 @@ import os from sqlalchemy import create_engine - +from sqlalchemy.pool import QueuePool class DatabaseEngine(): """Helper for database connections using SQLAlchemy engines""" @@ -10,17 +10,23 @@ def __init__(self): """Constructor""" self.engines = {} - def db_engine(self, conn_str): + def db_engine(self, conn_str, pool_size=10, max_overflow=20, pool_timeout=30, pool_recycle=300): """Return engine. :param str conn_str: DB connection string for SQLAlchemy engine - see http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql + see https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql """ engine = self.engines.get(conn_str) if not engine: engine = create_engine( - conn_str, pool_pre_ping=True, echo=False) + conn_str, + poolclass=QueuePool, + pool_size=pool_size, + max_overflow=max_overflow, + pool_timeout=pool_timeout, + pool_recycle=pool_recycle, + pool_pre_ping=True, echo=False) self.engines[conn_str] = engine return engine From f41bbaea4bce5986ffd56d4e64cfbcea869a8643 Mon Sep 17 00:00:00 2001 From: CebullaD Date: Fri, 14 Mar 2025 10:36:42 +0100 Subject: [PATCH 2/5] Connection pool parameters using ENV --- README.md | 4 ++++ qwc_services_core/database.py | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9a6f5ea..f9818d0 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ Environment variables | `TENANT_HEADER` | `` | The name of the HTTP header which contains the tenant name for multi-tenant setups. | | `TENANT_PATH_PREFIX` | `@service_prefix@/@tenant@` | URL path prefix for all QWC services for multi-tenant setups. | | `TENANT_ACCESS_COOKIE_PATH` | `` | Path for which the access cookie is valid for multi-tenant setups. | +| `POOL_SIZE` | `5` | Maximum number of possible data base connections. | +| `MAX_OVERFLOW` | `10` | Additional connections beyond pool_size during peak load. | +| `POOL_TIMEOUT` | `30` | Time (in seconds) to wait for a connection to become available. | +| `POOL_RECYCLE` | `-1` | Time (in seconds) after idle connections will be resetted. | Development =========== diff --git a/qwc_services_core/database.py b/qwc_services_core/database.py index e53e89c..f0a55b5 100644 --- a/qwc_services_core/database.py +++ b/qwc_services_core/database.py @@ -10,22 +10,32 @@ def __init__(self): """Constructor""" self.engines = {} - def db_engine(self, conn_str, pool_size=10, max_overflow=20, pool_timeout=30, pool_recycle=300): + def db_engine(self, conn_str, pool_size=5, max_overflow=10, pool_timeout=30, pool_recycle=-1): """Return engine. :param str conn_str: DB connection string for SQLAlchemy engine + :param int pool_size: Maximum number of possible connections + :param int max_overflow: Additional connections beyond pool_size during peak load + :param int pool_timeout: Time (in seconds) to wait for a connection to become available + :param int pool_recycle: Time (in seconds) after idle connections will be resetted see https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql """ + + db_pool_size = self.db_engine_env('POOL_SIZE', pool_size) + db_max_overflow = self.db_engine_env('MAX_OVERFLOW', max_overflow) + db_pool_timeout = self.db_engine_env('POOL_TIMEOUT', pool_timeout) + db_pool_recycle = self.db_engine_env('POOL_RECYCLE', pool_recycle) + engine = self.engines.get(conn_str) if not engine: engine = create_engine( conn_str, poolclass=QueuePool, - pool_size=pool_size, - max_overflow=max_overflow, - pool_timeout=pool_timeout, - pool_recycle=pool_recycle, + pool_size=db_pool_size, + max_overflow=db_max_overflow, + pool_timeout=db_pool_timeout, + pool_recycle=db_pool_recycle, pool_pre_ping=True, echo=False) self.engines[conn_str] = engine return engine @@ -36,11 +46,11 @@ def db_engine_env(self, env_name, default=None): :param str env_name: Environment variable :param str default: Default value if environment variable is not set """ - conn_str = os.environ.get(env_name, default) - if conn_str is None: + env_value = os.environ.get(env_name, default) + if env_value is None: raise Exception( 'db_engine_env: Environment variable %s not set' % env_name) - return self.db_engine(conn_str) + return self.db_engine(env_value) def geo_db(self): """Return engine for default GeoDB.""" From 094e5a2b05b2a8f74e3157ae8a78951a5665ecbc Mon Sep 17 00:00:00 2001 From: CebullaD Date: Mon, 17 Mar 2025 09:31:54 +0100 Subject: [PATCH 3/5] Set ENV default values directly and revert misleading use of db_engine_env() --- qwc_services_core/database.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/qwc_services_core/database.py b/qwc_services_core/database.py index f0a55b5..c485136 100644 --- a/qwc_services_core/database.py +++ b/qwc_services_core/database.py @@ -10,7 +10,7 @@ def __init__(self): """Constructor""" self.engines = {} - def db_engine(self, conn_str, pool_size=5, max_overflow=10, pool_timeout=30, pool_recycle=-1): + def db_engine(self, conn_str): """Return engine. :param str conn_str: DB connection string for SQLAlchemy engine @@ -22,10 +22,10 @@ def db_engine(self, conn_str, pool_size=5, max_overflow=10, pool_timeout=30, poo see https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql """ - db_pool_size = self.db_engine_env('POOL_SIZE', pool_size) - db_max_overflow = self.db_engine_env('MAX_OVERFLOW', max_overflow) - db_pool_timeout = self.db_engine_env('POOL_TIMEOUT', pool_timeout) - db_pool_recycle = self.db_engine_env('POOL_RECYCLE', pool_recycle) + db_pool_size = os.environ.get('POOL_SIZE', 5) + db_max_overflow = os.environ.get('MAX_OVERFLOW', 10) + db_pool_timeout = os.environ.get('POOL_TIMEOUT', 30) + db_pool_recycle = os.environ.get('POOL_RECYCLE', -1) engine = self.engines.get(conn_str) if not engine: @@ -46,11 +46,11 @@ def db_engine_env(self, env_name, default=None): :param str env_name: Environment variable :param str default: Default value if environment variable is not set """ - env_value = os.environ.get(env_name, default) - if env_value is None: + conn_str = os.environ.get(env_name, default) + if conn_str is None: raise Exception( 'db_engine_env: Environment variable %s not set' % env_name) - return self.db_engine(env_value) + return self.db_engine(conn_str) def geo_db(self): """Return engine for default GeoDB.""" From b89ca6a3ef2d5a075ab005caaad8286ec13f3caa Mon Sep 17 00:00:00 2001 From: Sandro Mani Date: Wed, 19 Mar 2025 09:59:40 +0100 Subject: [PATCH 4/5] Remove obsolete docstring --- qwc_services_core/database.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/qwc_services_core/database.py b/qwc_services_core/database.py index c485136..e56a795 100644 --- a/qwc_services_core/database.py +++ b/qwc_services_core/database.py @@ -13,12 +13,6 @@ def __init__(self): def db_engine(self, conn_str): """Return engine. - :param str conn_str: DB connection string for SQLAlchemy engine - :param int pool_size: Maximum number of possible connections - :param int max_overflow: Additional connections beyond pool_size during peak load - :param int pool_timeout: Time (in seconds) to wait for a connection to become available - :param int pool_recycle: Time (in seconds) after idle connections will be resetted - see https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql """ From 5c8d0b2d7237f9826e698a3a00eaaf695d4e40c8 Mon Sep 17 00:00:00 2001 From: Sandro Mani Date: Wed, 19 Mar 2025 10:00:09 +0100 Subject: [PATCH 5/5] Update database.py --- qwc_services_core/database.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qwc_services_core/database.py b/qwc_services_core/database.py index e56a795..583d8d1 100644 --- a/qwc_services_core/database.py +++ b/qwc_services_core/database.py @@ -13,6 +13,8 @@ def __init__(self): def db_engine(self, conn_str): """Return engine. + :param str conn_str: DB connection string for SQLAlchemy engine + see https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql """