From 7cd0962e60d26c71e3e0952940fcf5a65aebcc47 Mon Sep 17 00:00:00 2001 From: Denis Patlazhan Date: Mon, 28 Apr 2025 11:11:27 -0700 Subject: [PATCH 1/2] add podman support add podman support cleanup lint add socket verification add scenarios and prevent early termination errors add Podman related instructions ruff lint add macos target to github actions fixing typo add docker and podman runtimes to gh actions workflow switch to docker-ce on ubuntu split test on podam and docker into 2 separate jobs split podman and docker tests on macos Podman and docker can't coexist on macos. thus running it separately fixing path switch to colima to run docker cli handle podman startup only test podman on MacOs removing mess remove macos and separate podman and docker on ubuntu adjust path to tests bogus commit to rerun the workflow fix freaking twine error typo indentation indentation --- .github/workflows/tests.yml | 101 +++++++++++++++++---- README.rst | 9 ++ dbtesttools/engines/postgres.py | 22 ++++- dbtesttools/tests/test_pgfixture.py | 66 ++++++++++++++ dbtesttools/tests/test_pgfixture_docker.py | 9 ++ dbtesttools/tests/test_pgfixture_podman.py | 9 ++ 6 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 dbtesttools/tests/test_pgfixture.py create mode 100644 dbtesttools/tests/test_pgfixture_docker.py create mode 100644 dbtesttools/tests/test_pgfixture_podman.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5e0f18..520d2e3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,30 +1,99 @@ -name: Run tests +name: Run parallel tests -on: ["push", "pull_request"] +on: [push, pull_request] jobs: + docker-tests: + name: Docker Tests / Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v3 - test: + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch + + - name: Install Docker CE + run: | + sudo apt-get update + sudo apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + sudo systemctl start docker + docker --version + + - name: Run Docker scenario tests + run: | + hatch run ci -- dbtesttools.tests.test_pgfixture_docker dbtesttools.tests.test_isolation + + podman-tests: + name: Podman Tests / Python ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install hatch + - name: Install Podman + run: | + sudo apt-get update + sudo apt-get install -y podman + podman --version - - name: Run tests - run: | - hatch run ci + - name: Start Podman API (rootless) + run: | + export XDG_RUNTIME_DIR="/run/user/$(id -u)" + sudo mkdir -p "$XDG_RUNTIME_DIR" + sudo chown $(id -u):$(id -g) "$XDG_RUNTIME_DIR" + mkdir -p "$XDG_RUNTIME_DIR" + nohup podman system service --time=0 unix://$XDG_RUNTIME_DIR/podman/podman.sock > podman-api.log 2>&1 & + echo "Waiting for podman.sock..." + for i in {1..10}; do + if podman info > /dev/null 2>&1; then + echo "Podman is up" + break + fi + sleep 2 + done + cat podman-api.log + if ! podman info > /dev/null 2>&1; then + echo "::error ::Podman API service failed to start" + cat podman-api.log + exit 1 + fi + - name: Run Podman scenario tests + run: | + hatch run ci -- dbtesttools.tests.test_pgfixture_podman dbtesttools.tests.test_isolation diff --git a/README.rst b/README.rst index b6c6e9b..7d3eae1 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,15 @@ PG containers. You will need to do up to two extra things: can set the DBTESTTOOLS_PG_IP_ADDR environment variable. +If you want to use Podman instead of Docker engine you will need to follow +these steps: +1. Enable and start podman on your host +```bash +systemctl --user enable podman +systemctl --user start podman +``` +2. `export DBTESTTOOLS_USE_PODMAN=1` variable. + This code has been in use daily on a large project at Cisco for a few years now, and is very stable. diff --git a/dbtesttools/engines/postgres.py b/dbtesttools/engines/postgres.py index 44614f2..f3e54a5 100644 --- a/dbtesttools/engines/postgres.py +++ b/dbtesttools/engines/postgres.py @@ -90,6 +90,26 @@ def has_savepoint(self): def setUp(self): """Do all the work to bring up a working Postgres fixture.""" super().setUp() + # Podman integration: optionally override Docker socket + if os.getenv("DBTESTTOOLS_USE_PODMAN") == "1": + if sys.platform == "darwin": + # MacOS Podman socket + podman_socket = ( + f"unix://{os.getenv('HOME')}" + f"/.local/share/containers/podman/machine/podman.sock" + ) + else: + # Linux Podman socket + podman_socket = ( + f"unix:///run/user/{os.getuid()}/podman/podman.sock" + ) + if os.path.exists(podman_socket.replace("unix://", "")): + os.environ["DOCKER_HOST"] = podman_socket + else: + raise FileNotFoundError( + f"Podman socket not found at {podman_socket}" + ) + self.client = docker.from_env() self.pull_image() self.find_free_port() @@ -156,7 +176,7 @@ def set_up_test_database(self): cur.close() c.close() - @retry(psycopg2.OperationalError, tries=30, delay=1) + @retry(psycopg2.OperationalError, tries=60, delay=1) def wait_for_pg_start(self): c = psycopg2.connect( "user='postgres' host='{ip}' port='{port}'" diff --git a/dbtesttools/tests/test_pgfixture.py b/dbtesttools/tests/test_pgfixture.py new file mode 100644 index 0000000..1c1b1b6 --- /dev/null +++ b/dbtesttools/tests/test_pgfixture.py @@ -0,0 +1,66 @@ +import os +import unittest +import warnings + +import testscenarios +import testtools +from sqlalchemy import text + +from dbtesttools.engines.postgres import PostgresContainerFixture + +warnings.filterwarnings("ignore", category=UserWarning, module="urllib3") + + +class TestPostgresContainer( + testscenarios.TestWithScenarios, testtools.TestCase +): + """Test that we can bring up a Postgres container.""" + + scenarios = [ + ("docker", {"podman": False}), + ("podman", {"podman": True}), + ] + + def setUp(self): + self.pg_fixture = None + super().setUp() + if self.podman: + os.environ["DBTESTTOOLS_USE_PODMAN"] = "1" + else: + os.environ.pop("DBTESTTOOLS_USE_PODMAN", None) + try: + fixture = PostgresContainerFixture(future=True) + fixture.setUp() + self.pg_fixture = fixture + except Exception as e: + print(f'[ERROR] Failed to start Postgres fixture: {e}') + + def tearDown(self): + if self.pg_fixture is not None: + if hasattr(self.pg_fixture, "engine"): + try: + self.pg_fixture.engine.dispose() + except Exception as e: + print(f"Warning: failed to dispose engine: {e}") + if hasattr(self.pg_fixture, "container"): + try: + self.pg_fixture.container.kill() + except Exception as e: + print(f"Warning: failed to kill container: {e}") + super().tearDown() + + def test_connection(self): + if self.pg_fixture is None: + self.fail("Postgres fixture was not initialized") + # Actually test that the database is reachable + conn = self.pg_fixture.connect() + try: + result = conn.execute(text("SELECT 1;")) + value = result.scalar() + self.assertEqual(value, 1) + finally: + conn.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/dbtesttools/tests/test_pgfixture_docker.py b/dbtesttools/tests/test_pgfixture_docker.py new file mode 100644 index 0000000..e1169a0 --- /dev/null +++ b/dbtesttools/tests/test_pgfixture_docker.py @@ -0,0 +1,9 @@ +from .test_pgfixture import TestPostgresContainer + +# Override the scenarios to only run Docker +TestPostgresContainer.scenarios = [ + ("docker", {"podman": False}), +] + +# Optional: re-export the test class to make unittest discovery happy +__all__ = ["TestPostgresContainer"] diff --git a/dbtesttools/tests/test_pgfixture_podman.py b/dbtesttools/tests/test_pgfixture_podman.py new file mode 100644 index 0000000..d5df3d2 --- /dev/null +++ b/dbtesttools/tests/test_pgfixture_podman.py @@ -0,0 +1,9 @@ +from .test_pgfixture import TestPostgresContainer + +# Override the scenarios to only run Podman +TestPostgresContainer.scenarios = [ + ("podman", {"podman": True}), +] + +# Optional: re-export the test class to make unittest discovery happy +__all__ = ["TestPostgresContainer"] From a2f445acc8a227f4e085fe501c23f5eb2a6c8f24 Mon Sep 17 00:00:00 2001 From: Julian Edwards Date: Tue, 6 May 2025 13:48:34 +1000 Subject: [PATCH 2/2] Simplify workflow tests --- .github/workflows/tests.yml | 86 ++-------------------- README.rst | 13 ++-- dbtesttools/tests/test_pgfixture_docker.py | 9 --- dbtesttools/tests/test_pgfixture_podman.py | 9 --- 4 files changed, 15 insertions(+), 102 deletions(-) delete mode 100644 dbtesttools/tests/test_pgfixture_docker.py delete mode 100644 dbtesttools/tests/test_pgfixture_podman.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 520d2e3..233d137 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,8 +3,8 @@ name: Run parallel tests on: [push, pull_request] jobs: - docker-tests: - name: Docker Tests / Python ${{ matrix.python-version }} + test: + name: Tests / Python ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: matrix: @@ -12,88 +12,18 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies run: | + sudo apt-get update && sudo apt-get -y install podman + podman --version + systemctl --user enable podman + systemctl --user start podman python -m pip install --upgrade pip pip install hatch - - - name: Install Docker CE - run: | - sudo apt-get update - sudo apt-get install -y \ - ca-certificates \ - curl \ - gnupg \ - lsb-release - sudo mkdir -p /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io - sudo systemctl start docker - docker --version - - - name: Run Docker scenario tests - run: | - hatch run ci -- dbtesttools.tests.test_pgfixture_docker dbtesttools.tests.test_isolation - - podman-tests: - name: Podman Tests / Python ${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install hatch - - - name: Install Podman - run: | - sudo apt-get update - sudo apt-get install -y podman - podman --version - - - name: Start Podman API (rootless) - run: | - export XDG_RUNTIME_DIR="/run/user/$(id -u)" - sudo mkdir -p "$XDG_RUNTIME_DIR" - sudo chown $(id -u):$(id -g) "$XDG_RUNTIME_DIR" - mkdir -p "$XDG_RUNTIME_DIR" - nohup podman system service --time=0 unix://$XDG_RUNTIME_DIR/podman/podman.sock > podman-api.log 2>&1 & - echo "Waiting for podman.sock..." - for i in {1..10}; do - if podman info > /dev/null 2>&1; then - echo "Podman is up" - break - fi - sleep 2 - done - cat podman-api.log - if ! podman info > /dev/null 2>&1; then - echo "::error ::Podman API service failed to start" - cat podman-api.log - exit 1 - fi - - - name: Run Podman scenario tests + - name: Run tests run: | - hatch run ci -- dbtesttools.tests.test_pgfixture_podman dbtesttools.tests.test_isolation + hatch run ci diff --git a/README.rst b/README.rst index 7d3eae1..ea05e55 100644 --- a/README.rst +++ b/README.rst @@ -94,12 +94,13 @@ PG containers. You will need to do up to two extra things: If you want to use Podman instead of Docker engine you will need to follow these steps: -1. Enable and start podman on your host -```bash -systemctl --user enable podman -systemctl --user start podman -``` -2. `export DBTESTTOOLS_USE_PODMAN=1` variable. + + 1. Enable and start podman on your host + ```bash + systemctl --user enable podman + systemctl --user start podman + ``` + 2. `export DBTESTTOOLS_USE_PODMAN=1` variable. This code has been in use daily on a large project at Cisco for a few years now, and is very stable. diff --git a/dbtesttools/tests/test_pgfixture_docker.py b/dbtesttools/tests/test_pgfixture_docker.py deleted file mode 100644 index e1169a0..0000000 --- a/dbtesttools/tests/test_pgfixture_docker.py +++ /dev/null @@ -1,9 +0,0 @@ -from .test_pgfixture import TestPostgresContainer - -# Override the scenarios to only run Docker -TestPostgresContainer.scenarios = [ - ("docker", {"podman": False}), -] - -# Optional: re-export the test class to make unittest discovery happy -__all__ = ["TestPostgresContainer"] diff --git a/dbtesttools/tests/test_pgfixture_podman.py b/dbtesttools/tests/test_pgfixture_podman.py deleted file mode 100644 index d5df3d2..0000000 --- a/dbtesttools/tests/test_pgfixture_podman.py +++ /dev/null @@ -1,9 +0,0 @@ -from .test_pgfixture import TestPostgresContainer - -# Override the scenarios to only run Podman -TestPostgresContainer.scenarios = [ - ("podman", {"podman": True}), -] - -# Optional: re-export the test class to make unittest discovery happy -__all__ = ["TestPostgresContainer"]