[#697] Github actions to run all PRC tests

.github/workflows/main.yml

@@ -2,6 +2,8 @@
 # https://github.com/actions/starter-workflows/blob/master/ci/python-package.yml
 # (C) Github, MIT License
 
+# Static type checking tests using `mypy`.
+
 name: "Python type checking"
 
 on: [push, pull_request]

.github/workflows/run-bats-tests.yml

@@ -0,0 +1,32 @@
+# Run a set of tests, each in its own container and with a potentially customized setup.
+# (Documentation and implementation for the test harness may be found in `irods/test/harness`.)
+
+name: run-bats-tests
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+          working-directory: ./irods/test/harness
+    strategy:
+      matrix:
+        python: ['3.9','3.13']
+        irods_server: ['4.3.4','5.0.2']
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build images
+        run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"
+
+      - name: run tests
+        run: |
+          for script in ../scripts/test[0-9]* ../login_auth_test_must_run_manually.py
+          do
+              ./docker_container_driver.sh -V $script
+          done

.github/workflows/run-local-suite.yml

@@ -0,0 +1,29 @@
+# Run the client test suite in a Docker container, targeting a locally running instance of the iRODS server.
+# (Documentation and implementation for the test harness may be found in `irods/test/harness`.)
+
+name: run-local-suite
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+          working-directory: ./irods/test/harness
+    strategy:
+      matrix:
+        python: ['3.9','3.13']
+        irods_server: ['4.3.4','5.0.2']
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build images
+        run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"
+
+      - name: run tests
+        run: |
+          ./docker_container_driver.sh -V ../scripts/run_suite_locally.sh

.github/workflows/run-the-tests.yml

@@ -0,0 +1,39 @@
+# Create a networked set of containers (via a Docker compose project) on which to run the client test suite.
+# (For further information, see the README in `docker-testing`.)
+
+name: run-the-tests
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+          working-directory: ./docker-testing
+    strategy:
+      matrix:
+        python: ['3.9','3.13']
+        irods_server: ['4.3.4','5.0.2']
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Start containers
+        run: ./start_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"
+
+      - name: run test
+        run: |
+          while :; do
+            client_container=$(docker ps --format "{{.Names}}"|grep python.client)
+            [ -n "$client_container" ] && break
+            sleep 1
+          done
+          echo "client_container = [$client_container]"
+          docker exec "${client_container}" /repo_root/docker-testing/run_tests.sh
+
+      - name: Stop containers
+        if: always()
+        run: ./stop_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"

Dockerfile.prc_test.centos

@@ -24,6 +24,5 @@ RUN  python${py_N} repo/docker_build/iinit.py \
         password rods
 SHELL ["/bin/bash","-c"]
 CMD  echo "Waiting on iRODS server... " ; \
-     python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \
      sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \
      newgrp irods < repo/run_python_tests.sh

Dockerfile.prc_test.ubuntu

@@ -31,6 +31,5 @@ SHELL ["/bin/bash","-c"]
 #  3. run python tests as the new group
 
 CMD  echo "Waiting on iRODS server... " ; \
-     python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \
      sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \
      newgrp irods < repo/run_python_tests.sh

docker-testing/README.md

@@ -0,0 +1,53 @@
+# A Topological Setup for Testing the Python Client
+
+The `docker-testing` directory contains the necessary files for building and
+running tests from the perspective of a specific client node in a larger network.
+
+We currently allow a choice of Python interpreter and iRODS server to be installed
+on the client and provider nodes of a simulated network topology.
+
+The choice of versions are dictated when running the test:
+
+|:------------------:|:---------------:|
+|Environment Variable| Valid Range     |
+|:-------------------|-----------------|
+IRODS_PACKAGE_VERSION|4.3.1 to 5.0.2   |
+PYTHON_VERSION       |3.9 to 3.13      |
+|:-------------------|-----------------|
+
+Currently the database server is fixed as Postgres.
+
+## Details of usage
+
+The file `$REPO/.github/workflows/run-the-tests.yml`
+(where `$REPO` is the `/path/to/local/python-irodsclient` repository)
+contains commands for starting the server and client containers and running the PRC
+suite in response to a push or pull-request.
+
+The test suite can also be run on any workstation with docker compose installed.
+What follows is a short summary of how to run the test configuration at the bench.
+It is this procedure which is run within the Github workflows.
+
+   1. Change the working directory to the REPO top directory, e.g.:
+      ```
+      cd /path/to/python-irodsclient
+      ```
+
+   2. Run:
+      ```
+      ./docker-testing/start_containers.sh 4.3.4 3.11
+      ```
+      This builds and runs the docker images for the project,  with "4.3.4" being the iRODS
+      version installed on the provider and "3.11" is the version of python run on the client side.
+
+   3. Run:
+      ```
+      docker exec <name-of-python-client-container> /repo_root/docker-testing/run_tests.sh
+      ```
+      (Note:  `/repo_root` is an actual literal path, internal to the container.)
+      You'll see the test output displayed on the console.  At completion, xmlrunner outputs are in /tmp.
+
+   4. Tail docker logs to see the iRODS server log.
+      ```
+      docker logs -f <provider_container_name>
+      ```

docker-testing/harness-docker-compose-irods-4.yml

@@ -0,0 +1 @@
+harness-docker-compose.yml

docker-testing/harness-docker-compose-irods-5.yml

@@ -0,0 +1 @@
+harness-docker-compose.yml

docker-testing/harness-docker-compose.yml

@@ -0,0 +1,45 @@
+version: '3'
+
+services:
+    irods-catalog:
+        build:
+            context: irods_catalog_${irods_major}
+        # 5432 is exposed by default and can conflict with other postgres containers.
+        # When the metalnx-db service is no longer needed, this stanza can be removed.
+        ports:
+            - "5430:5432"
+        environment:
+            - POSTGRES_PASSWORD=testpassword
+
+    python-client:
+        build:
+            context: python_client
+            args:
+                python_version: ${python_version}
+        command:
+            tail -f /dev/null
+        volumes:
+            - ${repo_external}:/repo_root:ro
+            - /tmp/irods-client-share.py-${python_version}:/irods_shared
+        depends_on:
+            irods-catalog-provider:
+                condition: service_healthy
+
+    irods-catalog-provider:
+        volumes:
+            - /tmp/irods-client-share.py-${python_version}:/irods_shared
+        build:
+            context: irods_catalog_provider_${irods_major}
+            args:
+                irods_version: ${irods_version}
+        shm_size: 500mb
+        healthcheck:
+            test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"]
+            interval: 10s
+            timeout: 10s
+            retries: 3
+        ports:
+            - "1247:1247"
+        depends_on:
+            - irods-catalog
+

docker-testing/iinit.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+# This script creates the client environment to authenticate natively
+# as the 'rods' admin on the Docker node running the Python client tests.
+# Thus, we don't need irods-icommands to be installed on that node.
+
+from irods.password_obfuscation import encode
+import json
+import os
+import sys
+from os import chmod
+from os.path import expanduser,exists,join
+from getopt import getopt
+
+
+home_env_path = expanduser('~/.irods')
+env_file_path = join(home_env_path,'irods_environment.json')
+auth_file_path = join(home_env_path,'.irodsA')
+
+
+def do_iinit(host, port, user, zone, password):
+    if not exists(home_env_path):
+        os.makedirs(home_env_path)
+    else:
+        raise RuntimeError('~/.irods already exists')
+
+    with open(env_file_path,'w') as env_file:
+        json.dump ( { "irods_host": host,
+                      "irods_port": int(port),
+                      "irods_user_name": user,
+                      "irods_zone_name": zone  }, env_file, indent=4)
+    with open(auth_file_path,'w') as auth_file:
+        auth_file.write(encode(password))
+    chmod (auth_file_path,0o600)
+
+
+def get_kv_pairs_from_cmdline(*args):
+    arglist = list(args)
+    while arglist:
+        k = arglist.pop(0)
+        v = arglist.pop(0)
+        yield k,v
+
+
+if __name__ == '__main__':
+    args = sys.argv[1:]
+    dct = {k:v for k,v in get_kv_pairs_from_cmdline(*args)}
+    do_iinit(**dct)

docker-testing/irods_catalog_4/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:12
+
+COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh

docker-testing/irods_catalog_4/init-user-db.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Adapted from "Initialization script" in documentation for official Postgres dockerhub:
+#   https://hub.docker.com/_/postgres/
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE DATABASE "ICAT";
+    CREATE USER irods WITH PASSWORD 'testpassword';
+    GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods;
+EOSQL

docker-testing/irods_catalog_5/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:16
+
+COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh

docker-testing/irods_catalog_5/init-user-db.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Adapted from "Initialization script" in documentation for official Postgres dockerhub:
+#   https://hub.docker.com/_/postgres/
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE DATABASE "ICAT";
+    CREATE USER irods WITH PASSWORD 'testpassword';
+    GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods;
+    ALTER DATABASE "ICAT" OWNER TO irods
+EOSQL

docker-testing/irods_catalog_provider_4/Dockerfile

@@ -0,0 +1,55 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y \
+        apt-transport-https \
+        gnupg \
+        wget \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \
+    echo "deb [arch=amd64] https://packages.irods.org/apt/ focal main" | tee /etc/apt/sources.list.d/renci-irods.list
+
+RUN apt-get update && \
+    apt-get install -y \
+        libcurl4-gnutls-dev \
+        jq \
+        python3 \
+        python3-distro \
+        python3-jsonschema \
+        python3-pip \
+        python3-psutil \
+        python3-requests \
+        rsyslog \
+        unixodbc \
+        gawk \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+ARG irods_version=4.3.1
+ARG irods_package_version_suffix=-0~focal
+ARG irods_package_version=${irods_version}${irods_package_version_suffix}
+ARG irods_resource_plugin_version=${irods_version}.0${irods_package_version_suffix}
+
+RUN apt-get update && \
+    apt-get install -y \
+         irods-database-plugin-postgres=${irods_package_version} \
+         irods-runtime=${irods_package_version} \
+         irods-server=${irods_package_version} \
+         irods-icommands=${irods_package_version} \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+COPY setup-${irods_version}.input /
+RUN mv /setup-${irods_version}.input /irods_setup.input
+
+WORKDIR /
+COPY entrypoint.sh .
+RUN chmod u+x ./entrypoint.sh
+ENTRYPOINT ["./entrypoint.sh"]