Add recreate_schema utility with test

update precommit config
update documentation config

.config/pre-commit-config.yaml

@@ -8,18 +8,19 @@ repos:
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.4.2
     hooks:
       # Run the linter.
       - id: ruff
         args: [--config, .config/ruff.toml, --fix]
       # Run the formatter.
       - id: ruff-format
         args: [--config, .config/ruff.toml]
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.1
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.1.38
     hooks:
+      # Run the pip compile
       - id: pip-compile
         name: pip-compile requirements.txt
         files: pyproject.toml
-        args: [pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt]
+        args: [ pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt ]

documentation/.readthedocs.yaml

@@ -11,5 +11,6 @@ build:
   jobs:
     pre_build:
       - pip install --upgrade mkdocs
+      - pip install mkdocs-material
       - pip install pydoc-markdown
       - pydoc-markdown documentation/pydoc-markdown.yaml

documentation/pydoc-markdown.yaml

@@ -9,7 +9,7 @@ renderer:
   output_directory: docs
   mkdocs_config:
     site_name: HDX Python Database
-    theme: mkdocs
+    theme: material
     repo_url: "https://github.com/OCHA-DAP/hdx-python-database"
   markdown:
     source_linker:

requirements.txt

@@ -4,25 +4,25 @@
 #
 #    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
 #
-bcrypt==4.1.2
+bcrypt==4.1.3
     # via paramiko
 cffi==1.16.0
     # via
     #   cryptography
     #   pynacl
 cfgv==3.4.0
     # via pre-commit
-coverage[toml]==7.4.3
+coverage[toml]==7.5.1
     # via pytest-cov
-cryptography==42.0.5
+cryptography==42.0.6
     # via paramiko
 distlib==0.3.8
     # via virtualenv
-filelock==3.13.1
+filelock==3.14.0
     # via virtualenv
 greenlet==3.0.3
     # via sqlalchemy
-identify==2.5.35
+identify==2.5.36
     # via pre-commit
 iniconfig==2.0.0
     # via pytest
@@ -32,37 +32,37 @@ packaging==24.0
     # via pytest
 paramiko==3.4.0
     # via sshtunnel
-platformdirs==4.2.0
+platformdirs==4.2.1
     # via virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
-pre-commit==3.6.2
+pre-commit==3.7.0
     # via hdx-python-database (pyproject.toml)
 psycopg[binary]==3.1.18
     # via hdx-python-database (pyproject.toml)
 psycopg-binary==3.1.18
     # via psycopg
-pycparser==2.21
+pycparser==2.22
     # via cffi
 pynacl==1.5.0
     # via paramiko
-pytest==8.1.1
+pytest==8.2.0
     # via
     #   hdx-python-database (pyproject.toml)
     #   pytest-cov
-pytest-cov==4.1.0
+pytest-cov==5.0.0
     # via hdx-python-database (pyproject.toml)
 pyyaml==6.0.1
     # via pre-commit
-sqlalchemy==2.0.28
+sqlalchemy==2.0.30
     # via hdx-python-database (pyproject.toml)
 sshtunnel==0.4.0
     # via hdx-python-database (pyproject.toml)
-typing-extensions==4.10.0
+typing-extensions==4.11.0
     # via
     #   psycopg
     #   sqlalchemy
-virtualenv==20.25.1
+virtualenv==20.26.1
     # via pre-commit
 
 # The following packages are considered to be unsafe in a requirements file:

src/hdx/database/__init__.py

@@ -3,10 +3,11 @@
 import logging
 from typing import Any, Optional, Type, Union
 
-from sqlalchemy import create_engine
+from sqlalchemy import Engine, create_engine
 from sqlalchemy.ext.automap import automap_base
 from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
 from sqlalchemy.pool import NullPool
+from sqlalchemy.sql.ddl import CreateSchema, DropSchema
 
 from ._version import version as __version__  # noqa: F401
 from .dburi import get_connection_uri
@@ -123,6 +124,7 @@ def get_session(
         db_uri: str,
         table_base: Type[DeclarativeBase] = NoTZBase,
         reflect: bool = False,
+        engine: Engine = None,
     ) -> Session:
         """Gets SQLAlchemy session given url. Tables must inherit from Base in
         hdx.utilities.database unless base is defined. If reflect is True,
@@ -135,11 +137,13 @@ def get_session(
             db_uri (str): Connection URI
             table_base (Type[DeclarativeBase]): Base database table class. Defaults to NoTZBase.
             reflect (bool): Whether to reflect existing tables. Defaults to False.
+            engine (Engine): SQLAlchemy engine to use. Defaults to None (create engine).
 
         Returns:
             sqlalchemy.orm.Session: SQLAlchemy session
         """
-        engine = create_engine(db_uri, poolclass=NullPool, echo=False)
+        if not engine:
+            engine = create_engine(db_uri, poolclass=NullPool, echo=False)
         Session = sessionmaker(bind=engine)
         if reflect:
             Base = automap_base(declarative_base=table_base)
@@ -151,3 +155,26 @@ def get_session(
         session = Session()
         session.reflected_classes = classes
         return session
+
+    @staticmethod
+    def recreate_schema(
+        engine: Engine, db_uri: str, schema_name: str = "public"
+    ) -> None:
+        """Wipe and create empty schema in database using SQLAlchemy.
+
+        Args:
+            engine (Engine): SQLAlchemy engine to use.
+            db_uri (str): Connection URI
+            schema_name (str): Schema name. Defaults to "public".
+
+        Returns:
+            None
+        """
+        # Wipe and create an empty schema
+        with engine.connect() as connection:
+            connection.execute(
+                DropSchema(schema_name, cascade=True, if_exists=True)
+            )
+            connection.commit()
+            connection.execute(CreateSchema(schema_name, if_not_exists=True))
+            connection.commit()

tests/hdx/database/conftest.py

@@ -9,12 +9,12 @@ def mock_psycopg(monkeypatch):
     def connect(*args, **kwargs):
         if PsycopgConnection.connected:
 
-            class Connection:
+            class PGConnection:
                 @staticmethod
                 def close():
                     return
 
-            return Connection()
+            return PGConnection()
         else:
             PsycopgConnection.connected = True
             raise psycopg.OperationalError

tests/hdx/database/test_database.py

@@ -81,6 +81,30 @@ def close(self):
 
         monkeypatch.setattr(Database, "get_session", get_session)
 
+    @pytest.fixture(scope="function")
+    def mock_engine(self):
+        class SQAConnection:
+            def __enter__(self):
+                return self
+
+            def __exit__(self, exc_type, exc_value, traceback) -> None:
+                pass
+
+            @staticmethod
+            def execute(_):
+                return
+
+            @staticmethod
+            def commit():
+                return
+
+        class MockEngine:
+            @staticmethod
+            def connect():
+                return SQAConnection()
+
+        return MockEngine()
+
     def test_get_session(self, nodatabase):
         assert DBTestDate.__tablename__ == "db_test_date"
         with Database(
@@ -115,14 +139,15 @@ def test_get_reflect_session(self, database_to_reflect):
             # we don't have a timezone here
             assert row.date1 == datetime(1993, 9, 23, 14, 12, 56, 111000)
 
-    def test_get_session_ssh(self, mock_psycopg, mock_SSHTunnelForwarder):
+    def test_get_session_ssh(
+        self, mock_psycopg, mock_SSHTunnelForwarder, mock_engine
+    ):
+        db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase"
+        Database.recreate_schema(mock_engine, db_uri)
         with Database(
             ssh_host="mysshhost", **TestDatabase.params_pg
         ) as dbsession:
-            assert (
-                str(dbsession.bind.engine.url)
-                == "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase"
-            )
+            assert str(dbsession.bind.engine.url) == db_uri
         params = deepcopy(TestDatabase.params_pg)
         del params["password"]
         with Database(