From f1272c14e160d9c204d95e0e20cce58f41fe5c8a Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 15 Oct 2024 16:56:50 +1300 Subject: [PATCH 1/2] Output stderr in exception for pg_restore errors --- .config/pre-commit-config.yaml | 4 ++-- requirements.txt | 30 ++++++++++++++--------------- src/hdx/database/postgresql.py | 2 +- tests/hdx/database/test_database.py | 4 +++- 4 files changed, 21 insertions(+), 19 deletions(-) mode change 100755 => 100644 requirements.txt diff --git a/.config/pre-commit-config.yaml b/.config/pre-commit-config.yaml index 9247a65..e88dbca 100644 --- a/.config/pre-commit-config.yaml +++ b/.config/pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: - id: end-of-file-fixer - id: check-ast - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.9 hooks: # Run the linter. - id: ruff @@ -17,7 +17,7 @@ repos: - id: ruff-format args: [--config, .config/ruff.toml] - repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.2.33 + rev: 0.4.18 hooks: # Run the pip compile - id: pip-compile diff --git a/requirements.txt b/requirements.txt old mode 100755 new mode 100644 index ffd286a..d8d9409 --- a/requirements.txt +++ b/requirements.txt @@ -2,23 +2,23 @@ # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt bcrypt==4.2.0 # via paramiko -cffi==1.17.0 +cffi==1.17.1 # via # cryptography # pynacl cfgv==3.4.0 # via pre-commit -coverage==7.6.1 +coverage==7.6.3 # via pytest-cov -cryptography==43.0.0 +cryptography==43.0.1 # via paramiko -distlib==0.3.8 +distlib==0.3.9 # via virtualenv -filelock==3.15.4 +filelock==3.16.1 # via virtualenv -greenlet==3.0.3 +greenlet==3.1.1 # via sqlalchemy -identify==2.6.0 +identify==2.6.1 # via pre-commit iniconfig==2.0.0 # via pytest @@ -26,23 +26,23 @@ nodeenv==1.9.1 # via pre-commit packaging==24.1 # via pytest -paramiko==3.4.1 +paramiko==3.5.0 # via sshtunnel -platformdirs==4.2.2 +platformdirs==4.3.6 # via virtualenv pluggy==1.5.0 # via pytest -pre-commit==3.8.0 +pre-commit==4.0.1 # via hdx-python-database (pyproject.toml) -psycopg==3.2.1 +psycopg==3.2.3 # via hdx-python-database (pyproject.toml) -psycopg-binary==3.2.1 +psycopg-binary==3.2.3 # via psycopg pycparser==2.22 # via cffi pynacl==1.5.0 # via paramiko -pytest==8.3.2 +pytest==8.3.3 # via # hdx-python-database (pyproject.toml) # pytest-cov @@ -50,7 +50,7 @@ pytest-cov==5.0.0 # via hdx-python-database (pyproject.toml) pyyaml==6.0.2 # via pre-commit -sqlalchemy==2.0.32 +sqlalchemy==2.0.35 # via hdx-python-database (pyproject.toml) sshtunnel==0.4.0 # via hdx-python-database (pyproject.toml) @@ -58,5 +58,5 @@ typing-extensions==4.12.2 # via # psycopg # sqlalchemy -virtualenv==20.26.3 +virtualenv==20.26.6 # via pre-commit diff --git a/src/hdx/database/postgresql.py b/src/hdx/database/postgresql.py index 0c0b075..f3e3e05 100644 --- a/src/hdx/database/postgresql.py +++ b/src/hdx/database/postgresql.py @@ -87,7 +87,7 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: except subprocess.CalledProcessError as ex: command = " ".join(subprocess_params) raise PostgresError( - f"{command} failed. Return code: {process.returncode}" + f"{command} failed. Return code: {process.returncode}. Error: {process.stderr}" ) from ex for line in process.stdout.splitlines(): diff --git a/tests/hdx/database/test_database.py b/tests/hdx/database/test_database.py index 09f5f84..7ea82bc 100755 --- a/tests/hdx/database/test_database.py +++ b/tests/hdx/database/test_database.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from os import remove -from os.path import join +from os.path import exists, join from tempfile import gettempdir import pytest @@ -21,6 +21,8 @@ def prepare_fn(): return Database.prepare_views([date_view_params]) dbpath = join(gettempdir(), "test_database.db") + if exists(dbpath): + remove(dbpath) with Database( database=dbpath, port=None, From 046e34081aa2e8c44837390ed562559ee4922f94 Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 15 Oct 2024 17:15:35 +1300 Subject: [PATCH 2/2] Reflect views --- src/hdx/database/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hdx/database/__init__.py b/src/hdx/database/__init__.py index 8c2f84c..854c9ee 100644 --- a/src/hdx/database/__init__.py +++ b/src/hdx/database/__init__.py @@ -258,7 +258,9 @@ def create_session( engine = create_engine(db_uri, poolclass=NullPool, echo=False) if reflect: Base = automap_base(declarative_base=table_base) - Base.prepare(autoload_with=engine) + Base.prepare( + autoload_with=engine, reflection_options={"views": True} + ) table_base = Base else: table_base.metadata.create_all(engine)