Merge per-process engine caching into dev

.pre-commit-config.yaml

@@ -5,21 +5,21 @@
 exclude: '^(\.tox|ci/templates|\.bumpversion\.cfg)(/|$)'
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.11.4
     hooks:
       - id: isort
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
-    hooks:
-      - id: flake8
   - repo: https://github.com/psf/black
-    rev: 22.8.0
+    rev: 22.12.0
     hooks:
       - id: black
-        language_version: python3 # Should be a command that runs python3.6+
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        language_version: python3.8

CHANGELOG.rst

@@ -459,7 +459,11 @@ Changed
   `#942 <https://github.com/openego/eGon-data/issues/942>`_
 * Desaggregate industry demands to OSM areas and industrial sites
   `#1001 <https://github.com/openego/eGon-data/issues/1001>`_
+* SQLAlchemy `engine` objects created via :code:`egon.data.db.engine`
+  are now cached on a per process basis, so only one `engine` is ever
+  created for a single process. This fixes issue `#799`_.
 
+.. _#799: https://github.com/openego/eGon-data/issues/799
 
 Bug Fixes
 ---------

src/egon/data/db.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 import codecs
 import functools
+import os
 import time
 
 from psycopg2.errors import DeadlockDetected, UniqueViolation
@@ -38,8 +39,8 @@ def credentials():
     return configuration
 
 
-def engine():
-    """Engine for local database."""
+@functools.lru_cache(maxsize=None)
+def engine_for(pid):  # pylint: disable=unused-argument
     db_config = credentials()
     return create_engine(
         f"postgresql+psycopg2://{db_config['POSTGRES_USER']}:"
@@ -49,6 +50,11 @@ def engine():
     )
 
 
+def engine():
+    """Engine for local database."""
+    return engine_for(os.getpid())
+
+
 def execute_sql(sql_string):
     """Execute a SQL expression given as string.
 
@@ -128,7 +134,7 @@ def session_scope():
     try:
         yield session
         session.commit()
-    except:
+    except:  # noqa: E722 (This is OK, because of the immediate re-raise.)
         session.rollback()
         raise
     finally:
@@ -354,7 +360,11 @@ def commit(*args, **kwargs):
 
 
 def assign_gas_bus_id(dataframe, scn_name, carrier):
-    """Assigns bus_ids to points (contained in a dataframe) according to location
+    """Assign `bus_id`s to points according to location.
+
+    The points are taken from the given `dataframe` and the geometries by
+    which the `bus_id`s are assigned to them are taken from the
+    `grid.egon_gas_voronoi` table.
 
     Parameters
     ----------