diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40ad105d3..1c9fcf912 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,8 @@ # https://github.com/actions/starter-workflows/blob/master/ci/python-package.yml # (C) Github, MIT License +# Static type checking tests using `mypy`. + name: "Python type checking" on: [push, pull_request] diff --git a/.github/workflows/run-bats-tests.yml b/.github/workflows/run-bats-tests.yml new file mode 100644 index 000000000..7a004725b --- /dev/null +++ b/.github/workflows/run-bats-tests.yml @@ -0,0 +1,32 @@ +# Run a set of tests, each in its own container and with a potentially customized setup. +# (Documentation and implementation for the test harness may be found in `irods/test/harness`.) + +name: run-bats-tests + +on: [push, pull_request] + +jobs: + tests: + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./irods/test/harness + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + for script in ../scripts/test[0-9]* ../login_auth_test_must_run_manually.py + do + ./docker_container_driver.sh -V $script + done diff --git a/.github/workflows/run-local-suite.yml b/.github/workflows/run-local-suite.yml new file mode 100644 index 000000000..be3dc606f --- /dev/null +++ b/.github/workflows/run-local-suite.yml @@ -0,0 +1,29 @@ +# Run the client test suite in a Docker container, targeting a locally running instance of the iRODS server. +# (Documentation and implementation for the test harness may be found in `irods/test/harness`.) + +name: run-local-suite + +on: [push, pull_request] + +jobs: + tests: + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./irods/test/harness + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + ./docker_container_driver.sh -V ../scripts/run_suite_locally.sh diff --git a/.github/workflows/run-the-tests.yml b/.github/workflows/run-the-tests.yml new file mode 100644 index 000000000..7cf256a16 --- /dev/null +++ b/.github/workflows/run-the-tests.yml @@ -0,0 +1,39 @@ +# Create a networked set of containers (via a Docker compose project) on which to run the client test suite. +# (For further information, see the README in `docker-testing`.) + +name: run-the-tests + +on: [push, pull_request] + +jobs: + tests: + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./docker-testing + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start containers + run: ./start_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run test + run: | + while :; do + client_container=$(docker ps --format "{{.Names}}"|grep python.client) + [ -n "$client_container" ] && break + sleep 1 + done + echo "client_container = [$client_container]" + docker exec "${client_container}" /repo_root/docker-testing/run_tests.sh + + - name: Stop containers + if: always() + run: ./stop_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" diff --git a/Dockerfile.prc_test.centos b/Dockerfile.prc_test.centos index debed6d6d..c171fa62e 100644 --- a/Dockerfile.prc_test.centos +++ b/Dockerfile.prc_test.centos @@ -24,6 +24,5 @@ RUN python${py_N} repo/docker_build/iinit.py \ password rods SHELL ["/bin/bash","-c"] CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/Dockerfile.prc_test.ubuntu b/Dockerfile.prc_test.ubuntu index e8c958a85..79ed07e12 100644 --- a/Dockerfile.prc_test.ubuntu +++ b/Dockerfile.prc_test.ubuntu @@ -31,6 +31,5 @@ SHELL ["/bin/bash","-c"] # 3. run python tests as the new group CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/docker-testing/README.md b/docker-testing/README.md new file mode 100644 index 000000000..9a331977a --- /dev/null +++ b/docker-testing/README.md @@ -0,0 +1,53 @@ +# A Topological Setup for Testing the Python Client + +The `docker-testing` directory contains the necessary files for building and +running tests from the perspective of a specific client node in a larger network. + +We currently allow a choice of Python interpreter and iRODS server to be installed +on the client and provider nodes of a simulated network topology. + +The choice of versions are dictated when running the test: + +|:------------------:|:---------------:| +|Environment Variable| Valid Range | +|:-------------------|-----------------| +IRODS_PACKAGE_VERSION|4.3.1 to 5.0.2 | +PYTHON_VERSION |3.9 to 3.13 | +|:-------------------|-----------------| + +Currently the database server is fixed as Postgres. + +## Details of usage + +The file `$REPO/.github/workflows/run-the-tests.yml` +(where `$REPO` is the `/path/to/local/python-irodsclient` repository) +contains commands for starting the server and client containers and running the PRC +suite in response to a push or pull-request. + +The test suite can also be run on any workstation with docker compose installed. +What follows is a short summary of how to run the test configuration at the bench. +It is this procedure which is run within the Github workflows. + + 1. Change the working directory to the REPO top directory, e.g.: + ``` + cd /path/to/python-irodsclient + ``` + + 2. Run: + ``` + ./docker-testing/start_containers.sh 4.3.4 3.11 + ``` + This builds and runs the docker images for the project, with "4.3.4" being the iRODS + version installed on the provider and "3.11" is the version of python run on the client side. + + 3. Run: + ``` + docker exec /repo_root/docker-testing/run_tests.sh + ``` + (Note: `/repo_root` is an actual literal path, internal to the container.) + You'll see the test output displayed on the console. At completion, xmlrunner outputs are in /tmp. + + 4. Tail docker logs to see the iRODS server log. + ``` + docker logs -f + ``` diff --git a/docker-testing/harness-docker-compose-irods-4.yml b/docker-testing/harness-docker-compose-irods-4.yml new file mode 120000 index 000000000..db32ad86f --- /dev/null +++ b/docker-testing/harness-docker-compose-irods-4.yml @@ -0,0 +1 @@ +harness-docker-compose.yml \ No newline at end of file diff --git a/docker-testing/harness-docker-compose-irods-5.yml b/docker-testing/harness-docker-compose-irods-5.yml new file mode 120000 index 000000000..db32ad86f --- /dev/null +++ b/docker-testing/harness-docker-compose-irods-5.yml @@ -0,0 +1 @@ +harness-docker-compose.yml \ No newline at end of file diff --git a/docker-testing/harness-docker-compose.yml b/docker-testing/harness-docker-compose.yml new file mode 100644 index 000000000..e2d9d2b15 --- /dev/null +++ b/docker-testing/harness-docker-compose.yml @@ -0,0 +1,45 @@ +version: '3' + +services: + irods-catalog: + build: + context: irods_catalog_${irods_major} + # 5432 is exposed by default and can conflict with other postgres containers. + # When the metalnx-db service is no longer needed, this stanza can be removed. +# ports: +# - "5430:5432" + environment: + - POSTGRES_PASSWORD=testpassword + + python-client: + build: + context: python_client + args: + python_version: ${python_version} + command: + tail -f /dev/null + volumes: + - ${repo_external}:/repo_root:ro + - /tmp/irods-client-share.py-${python_version}:/irods_shared + depends_on: + irods-catalog-provider: + condition: service_healthy + + irods-catalog-provider: + volumes: + - /tmp/irods-client-share.py-${python_version}:/irods_shared + build: + context: irods_catalog_provider_${irods_major} + args: + irods_version: ${irods_version} + shm_size: 500mb + healthcheck: + test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"] + interval: 10s + timeout: 10s + retries: 3 +# ports: +# - "1247:1247" + depends_on: + - irods-catalog + diff --git a/docker-testing/iinit.py b/docker-testing/iinit.py new file mode 100755 index 000000000..776ca97ee --- /dev/null +++ b/docker-testing/iinit.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# This script creates the client environment to authenticate natively +# as the 'rods' admin on the Docker node running the Python client tests. +# Thus, we don't need irods-icommands to be installed on that node. + +from irods.password_obfuscation import encode +import json +import os +import sys +from os import chmod +from os.path import expanduser,exists,join +from getopt import getopt + + +home_env_path = expanduser('~/.irods') +env_file_path = join(home_env_path,'irods_environment.json') +auth_file_path = join(home_env_path,'.irodsA') + + +def do_iinit(host, port, user, zone, password): + if not exists(home_env_path): + os.makedirs(home_env_path) + else: + raise RuntimeError('~/.irods already exists') + + with open(env_file_path,'w') as env_file: + json.dump ( { "irods_host": host, + "irods_port": int(port), + "irods_user_name": user, + "irods_zone_name": zone }, env_file, indent=4) + with open(auth_file_path,'w') as auth_file: + auth_file.write(encode(password)) + chmod (auth_file_path,0o600) + + +def get_kv_pairs_from_cmdline(*args): + arglist = list(args) + while arglist: + k = arglist.pop(0) + v = arglist.pop(0) + yield k,v + + +if __name__ == '__main__': + args = sys.argv[1:] + dct = {k:v for k,v in get_kv_pairs_from_cmdline(*args)} + do_iinit(**dct) diff --git a/docker-testing/irods_catalog_4/Dockerfile b/docker-testing/irods_catalog_4/Dockerfile new file mode 100644 index 000000000..f02c4520c --- /dev/null +++ b/docker-testing/irods_catalog_4/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:12 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/docker-testing/irods_catalog_4/init-user-db.sh b/docker-testing/irods_catalog_4/init-user-db.sh new file mode 100644 index 000000000..5ff6b0375 --- /dev/null +++ b/docker-testing/irods_catalog_4/init-user-db.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; +EOSQL diff --git a/docker-testing/irods_catalog_5/Dockerfile b/docker-testing/irods_catalog_5/Dockerfile new file mode 100644 index 000000000..112ffbaaa --- /dev/null +++ b/docker-testing/irods_catalog_5/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:16 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/docker-testing/irods_catalog_5/init-user-db.sh b/docker-testing/irods_catalog_5/init-user-db.sh new file mode 100644 index 000000000..f3c724e2f --- /dev/null +++ b/docker-testing/irods_catalog_5/init-user-db.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; + ALTER DATABASE "ICAT" OWNER TO irods +EOSQL diff --git a/docker-testing/irods_catalog_provider_4/Dockerfile b/docker-testing/irods_catalog_provider_4/Dockerfile new file mode 100644 index 000000000..7632b273f --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/Dockerfile @@ -0,0 +1,55 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ focal main" | tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ARG irods_version=4.3.1 +ARG irods_package_version_suffix=-0~focal +ARG irods_package_version=${irods_version}${irods_package_version_suffix} +ARG irods_resource_plugin_version=${irods_version}.0${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +RUN chmod u+x ./entrypoint.sh +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker-testing/irods_catalog_provider_4/entrypoint.sh b/docker-testing/irods_catalog_provider_4/entrypoint.sh new file mode 100644 index 000000000..3a321db2f --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/entrypoint.sh @@ -0,0 +1,49 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q +do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py < "${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared + +#TODO ensure this is done for 4.3+ only. 4.2 doesn't have this server config key +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && \ + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && \ + touch ~/provider-address.do_not_remove + } +} || { echo >&2 "Error modifying $ORIG_SERVER_CONFIG"; exit 1; } + +echo "Starting server" + +cd /usr/sbin +su irods -c 'bash -c "./irodsServer -u"' + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.1.input b/docker-testing/irods_catalog_provider_4/setup-4.3.1.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.1.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.2.input b/docker-testing/irods_catalog_provider_4/setup-4.3.2.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.2.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.3.input b/docker-testing/irods_catalog_provider_4/setup-4.3.3.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.3.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.4.input b/docker-testing/irods_catalog_provider_4/setup-4.3.4.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.4.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_5/Dockerfile b/docker-testing/irods_catalog_provider_5/Dockerfile new file mode 100644 index 000000000..58daffd87 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/Dockerfile @@ -0,0 +1,59 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ noble main" | tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + postgresql-client-16 \ + vim-tiny \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +# - postgres client 16 is for pg_isready +# TODO delete vim-tiny + +ARG irods_version=5.0.1 +ARG irods_package_version_suffix=-0~noble +ARG irods_package_version=${irods_version}${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +RUN chmod u+x ./entrypoint.sh +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker-testing/irods_catalog_provider_5/entrypoint.sh b/docker-testing/irods_catalog_provider_5/entrypoint.sh new file mode 100644 index 000000000..7034760a1 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/entrypoint.sh @@ -0,0 +1,50 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q +do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py < "${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared +chmod 0777 /irods_shared + +#TODO ensure this is done for 4.3+ only. 4.2 doesn't have this server config key +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && \ + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && \ + touch ~/provider-address.do_not_remove + } +} || { echo >&2 "Error modifying $ORIG_SERVER_CONFIG"; exit 1; } + +echo "Starting server" + +cd /usr/sbin +su irods -c 'bash -c "./irodsServer -p /tmp/irods.pid --stdout"' + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.0.input b/docker-testing/irods_catalog_provider_5/setup-5.0.0.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.0.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.1.input b/docker-testing/irods_catalog_provider_5/setup-5.0.1.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.1.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.2.input b/docker-testing/irods_catalog_provider_5/setup-5.0.2.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.2.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/print_repo_root_location b/docker-testing/print_repo_root_location new file mode 100755 index 000000000..79d91af7e --- /dev/null +++ b/docker-testing/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/docker-testing/python_client/Dockerfile b/docker-testing/python_client/Dockerfile new file mode 100644 index 000000000..83c7ff885 --- /dev/null +++ b/docker-testing/python_client/Dockerfile @@ -0,0 +1,3 @@ +ARG python_version +FROM python:${python_version} +RUN pip install remote-pdb diff --git a/docker-testing/run_tests.sh b/docker-testing/run_tests.sh new file mode 100755 index 000000000..8ed4a77af --- /dev/null +++ b/docker-testing/run_tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e -x +PYTHON=$(which python3) +if [ -z "$PYTHON" ]; then + PYTHON=$(which python) +fi +DIR=$(dirname "$0") +cd "$DIR" + +REPO="$(./print_repo_root_location)" + +if [ -d /irods_shared ]; then + + # Get the numeric user and group id's for irods service account on the provider. This helps to set up the test user + # (named 'user') with proper permissions for the shared volume on the client node. + groupadd -o -g $(stat -c%g /irods_shared) irods + useradd -g irods -u $(stat -c%u /irods_shared) irods + + # Set up useful subdirectories in the client/provider shared volume. + mkdir /irods_shared/{tmp,reg_resc} + chown irods:irods /irods_shared/{tmp,reg_resc} + chmod 777 /irods_shared/reg_resc + chmod g+ws /irods_shared/tmp + + # Make a test user in group irods, who will run the client tests. + useradd -G irods -m -s/bin/bash user + + # Create writable copy of this repo. + cp -r /"$REPO"{,.copy} + REPO+=.copy + chown -R user "$REPO" + chmod u+w "$REPO"/irods/test/test-data + + # Install PRC from the repo. + $PYTHON -m pip install "$REPO[tests]" +fi + +su - user -c "\ +$PYTHON '$DIR'/iinit.py \ + host irods-catalog-provider \ + port 1247 \ + user rods \ + password rods \ + zone tempZone +$PYTHON '$REPO'/irods/test/runner.py $*" diff --git a/docker-testing/start_containers.sh b/docker-testing/start_containers.sh new file mode 100755 index 000000000..b4b748cb3 --- /dev/null +++ b/docker-testing/start_containers.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 [-n] [-b ""] [irods_version] python_version"; exit 2; +} + +SHELL_DOCKER_COMPOSE_BUILD_ARGS="" +DO_NOT_RUN="" + +while [[ $1 = -* ]]; do + if [ "$1" = "-b" ]; then + SHELL_DOCKER_COMPOSE_BUILD_ARGS=$2 + shift 2 + fi + if [ "$1" = "-n" ]; then + DO_NOT_RUN=1 + shift + fi +done + +if [ $# -eq 2 ]; then + IRODS_VERSION=$1 + PYTHON_VERSION=$2 +elif [ $# -eq 1 ]; then + IRODS_VERSION=4.3.4 + PYTHON_VERSION=$1 +else + usage +fi + +shift $# + +[ -n "$PYTHON_VERSION" -a -n "$IRODS_VERSION" ] || { + usage +} + +IRODS_MAJOR=${IRODS_VERSION//.*/} + +DIR=$(dirname "$0") +cd "${DIR}" +REPO_ROOT=$(realpath ..) + +echo "\ +repo_external=\"${REPO_ROOT}\" +python_version=\"${PYTHON_VERSION}\" +irods_version=\"${IRODS_VERSION}\" +irods_major=\"${IRODS_MAJOR}\"" >.env + +# In case the docker-compose setup varies between iRODS major releases, the .YML file may be a symbolic link. + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml build $SHELL_DOCKER_COMPOSE_BUILD_ARGS + +if [ -z "$DO_NOT_RUN" ]; then + docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml up -d +fi diff --git a/docker-testing/stop_containers.sh b/docker-testing/stop_containers.sh new file mode 100755 index 000000000..c41002287 --- /dev/null +++ b/docker-testing/stop_containers.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 [irods_version] python_version"; exit 1; +} + +if [ $# -eq 2 ]; then + IRODS_VERSION=$1 + PYTHON_VERSION=$2 +elif [ $# -eq 1 ]; then + IRODS_VERSION=4.3.4 + PYTHON_VERSION=$1 +else + usage +fi + +shift $# + +[ -n "$PYTHON_VERSION" -a -n "$IRODS_VERSION" ] || { + usage +} + +IRODS_MAJOR=${IRODS_VERSION//.*/} + +# In case the docker-compose setup varies between iRODS major releases, the .YML file may be a symbolic link. + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml down diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 818ae3677..9b8e0ec80 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -181,7 +181,7 @@ def ET(xml_type=(), server_version=None): logger = logging.getLogger(__name__) -IRODS_VERSION = (5, 0, 1, "d") +IRODS_VERSION = (5, 0, 2, "d") UNICODE = str diff --git a/irods/test/access_test.py b/irods/test/access_test.py index 7c496c685..73c122b76 100644 --- a/irods/test/access_test.py +++ b/irods/test/access_test.py @@ -533,11 +533,7 @@ def test_iRODSAccess_cannot_be_constructed_using_unsupported_type__issue_558(sel # Before the fix in #558, this would have been allowed and only later would the type discrepancy be revealed, # leading to opaque error messages. Now, the types are checked on the way in to ensure clarity and correctness. # TODO(#480): We cannot use the unittest.assertRaises context manager as this was introduced in python 3.1. - assertCall = getattr(self, "assertRaisesRegex", None) - if assertCall is None: - assertCall = self.assertRaisesRegexp - - assertCall( + self.assertRaisesRegex( TypeError, "'path' parameter must be of type 'str', 'irods.collection.iRODSCollection', " "'irods.data_object.iRODSDataObject', or 'irods.path.iRODSPath'.", diff --git a/irods/test/data_obj_test.py b/irods/test/data_obj_test.py index b6ed95886..6159a220a 100644 --- a/irods/test/data_obj_test.py +++ b/irods/test/data_obj_test.py @@ -3305,16 +3305,13 @@ def test_access_time__issue_700(self): if self.sess.server_version < (5,): self.skipTest("iRODS servers < 5.0.0 do not provide an access_time attribute for data objects.") - data_path= iRODSPath(self.coll.path, - unique_name(my_function_name(), datetime.now()) - ) - with self.sess.data_objects.open(data_path,"w") as f: - f.write(b'_') - with self.sess.data_objects.open(data_path,"r") as f: - f.read() + # Create a new, uniquely named test data object. + data = self.sess.data_objects.create( + f'{helpers.home_collection(self.sess)}/{unique_name(my_function_name(), datetime.now())}' + ) - data = self.sess.data_objects.get(data_path) - self.assertGreaterEqual(data.access_time, data.modify_time) + # Test that access_time is there, and of the right type. + self.assertIs(type(data.access_time), datetime) if __name__ == "__main__": # let the tests find the parent irods lib diff --git a/irods/test/exception_test.py b/irods/test/exception_test.py index 16a95c989..b400cf2aa 100644 --- a/irods/test/exception_test.py +++ b/irods/test/exception_test.py @@ -41,8 +41,8 @@ def test_400(self): excep_repr = repr(exc) errno_object = irods.exception.Errno(errno.EACCES) errno_repr = repr(errno_object) - self.assertRegexpMatches(errno_repr, r"\bErrno\b") - self.assertRegexpMatches( + self.assertRegex(errno_repr, r"\bErrno\b") + self.assertRegex( errno_repr, """['"]{msg}['"]""".format(msg=os.strerror(errno.EACCES)) ) self.assertIn(errno_repr, excep_repr) diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile new file mode 100644 index 000000000..d10f7583b --- /dev/null +++ b/irods/test/harness/000_install-irods.Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:22.04 +COPY install.sh / +ARG irods_package_version +ENV IRODS_PACKAGE_VERSION "$irods_package_version" +RUN for phase in initialize install-essential-packages add-package-repo; do \ + bash /install.sh --w=$phase 0; \ + done +RUN /install.sh 4 +COPY start_postgresql_and_irods.sh manage_irods5_procs / +RUN apt install -y sudo +RUN useradd -ms/bin/bash testuser +RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers +RUN apt install -y faketime +ENV IRODS_CONTROL_PATH="" +CMD bash $IRODS_CONTROL_PATH/start_postgresql_and_irods.sh diff --git a/irods/test/harness/001_bats-python3.Dockerfile b/irods/test/harness/001_bats-python3.Dockerfile new file mode 100644 index 000000000..b78179856 --- /dev/null +++ b/irods/test/harness/001_bats-python3.Dockerfile @@ -0,0 +1,5 @@ +FROM install-irods +RUN apt update; apt install -y python3-pip bats +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install virtualenv +RUN python3 -m virtualenv /py3 diff --git a/irods/test/harness/002_ssl-and-pam.Dockerfile b/irods/test/harness/002_ssl-and-pam.Dockerfile new file mode 100644 index 000000000..810d10e86 --- /dev/null +++ b/irods/test/harness/002_ssl-and-pam.Dockerfile @@ -0,0 +1 @@ +FROM bats-python3 diff --git a/irods/test/harness/003_compile-specific-python.Dockerfile b/irods/test/harness/003_compile-specific-python.Dockerfile new file mode 100644 index 000000000..63f3ae18c --- /dev/null +++ b/irods/test/harness/003_compile-specific-python.Dockerfile @@ -0,0 +1,16 @@ +FROM ssl-and-pam +RUN apt update +RUN apt install -y wget build-essential +RUN apt install -y libssl-dev zlib1g-dev libffi-dev libncurses-dev wget build-essential +ARG python_version +RUN wget https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tar.xz +RUN tar xf Python-${python_version}.tar.xz +WORKDIR /Python-${python_version} +RUN ./configure --prefix /root/python --with-ensurepip=install +RUN make -j +RUN mkdir /root/python +RUN make install +WORKDIR / +RUN /root/python/bin/python3 -m pip install virtualenv +RUN chmod a+rx /root +ENV PYTHON_VERSION=${python_version} diff --git a/irods/test/harness/README.md b/irods/test/harness/README.md new file mode 100644 index 000000000..de4a33b36 --- /dev/null +++ b/irods/test/harness/README.md @@ -0,0 +1,60 @@ +# Docker Powered Test Harness + +## Description + +A series of docker images which support running isolated test scripts (using BATS, bash, or Python). +Once built, the images allow loading and customizing the Docker container environment for a given +test script. + +The general form for test invocation is: `docker_container_driver.sh ` + +Within the container, a computed internal path to the same script is executed, whether directly or +indirectly by a wrapper script. The wrapper for many of the PRC authentication-via-PAM tests is +irods/test/login_auth_test.sh. + +The test_script_parameters file, located in the irods/test/harness directory, contains customized +settings for each test script run, including: + + - Docker image name to be used. + + - Wrapper to be invoked, if any. Wrappers shall perform common setup tasks up to and including + invoking the test script itself. + + - Which user is running the test. (Unless otherwise specified, this is the passwordless-sudo- + enabled user `testuser`). + +When done with a test, the `docker_container_driver.sh` exit code mirrors the return code from the +run of the test script. The container itself is removed unless the `-L` ("leak") option is given. + +## Sample Runs + +### To build required images + +For our convenience in this doc, set a shell variable `REPOROOT` to `~/python-irodsclient` (or +similar) to specify the path to the top level of the local repository. + +Sample command lines to build Docker images: + +1. ``` + cd $REPO_ROOT/irods/test/harness + ./build_docker.sh + ``` + + Builds docker images in proper sequence. + +2. ``` + cd $REPO_ROOT/irods/test/harness; + IRODS_PACKAGE_VERSION=4.3.4 PYTHON_VERSION=3.11 NO_CACHE=1 ./build-docker.sh [ Dockerfiles... ] + ``` + + Builds (ignoring docker cache) images based on specific iRODS package version and desired + Python Interpreter version, optionally with a restricted list of Docker files in need of rebulding. + +### To run a test script. + +``` +$REPO_ROOT/irods/test/harness/docker_container_driver.sh $REPO_ROOT/irods/test/scripts/run_local_suite +``` + +For both builder and driver script, the environment variable `DOCKER` may be set to `podman` to run +the alternative container engine. Otherwise it default to a value of `docker`. diff --git a/irods/test/harness/build-docker.sh b/irods/test/harness/build-docker.sh new file mode 100755 index 000000000..9c1072692 --- /dev/null +++ b/irods/test/harness/build-docker.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# environment variables for build +# IRODS_PACKAGE_VERSION if defined is like "4.3.4" or "5.0.1". +# (but contains no '~' suffix for irods versions <= 4.2.10) +# PYTHON_VERSION is usually two dot-separated numbers: example "3.13", but could also have zero, one or three version numbers. +# (Do not specify the triple form, X.Y.Z, if that release is not known to exist - not counting alphas and release candidates) + +DIR=$(realpath "$(dirname "$0")") +: ${DOCKER:=docker} + +if [ $# -gt 0 ]; then + IFS=$'\n' read -ra ARGS -d '' < <(realpath --relative-to "$DIR" "$@") + cd "$DIR" +else + cd "$DIR" + ARGS=([0-9]*.Dockerfile) +fi + +: ${PYTHON_VERSION:=3.13} export PYTHON_VERSION + +for dockerfile in "${ARGS[@]}"; do + image_name=${dockerfile#[0-9]*_} + image_name=${image_name%.Dockerfile} + irods_package_version_option="" + python_version_option="" + if [ "$image_name" = "install-irods" ]; then + irods_package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"} + elif [ "$image_name" = "compile-specific-python" ]; then + temp=$(./most_recent_python.sh "$PYTHON_VERSION") + if [ -n "$temp" ]; then + PYTHON_VERSION="$temp" + fi + python_version_option=${PYTHON_VERSION:+"--build-arg=python_version=$PYTHON_VERSION"} + fi + $DOCKER build -f $dockerfile -t $image_name . $irods_package_version_option $python_version_option \ + ${NO_CACHE+"--no-cache"} || + { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; } +done diff --git a/irods/test/harness/create_docker_images.sh b/irods/test/harness/create_docker_images.sh new file mode 100755 index 000000000..2f831a078 --- /dev/null +++ b/irods/test/harness/create_docker_images.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export IRODS_PACKAGE_VERSION=$1 +export PYTHON_VERSION=$2 + +[ -z "$1" -o -z "$2" ] && { + echo >&2 "usage $0 irods-vsn py-vsn"; exit 2; +} +shift 2 + +DIR=$(dirname "$0") + +"$DIR"/build-docker.sh $* diff --git a/irods/test/harness/docker_container_driver.sh b/irods/test/harness/docker_container_driver.sh new file mode 100755 index 000000000..6bfcd5457 --- /dev/null +++ b/irods/test/harness/docker_container_driver.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +# Runs a test program within a new container. The container is dispatched and/or disposed of, and the exit +# status code of the target test program collected and returned, by this script. + +# The repository containing this harness directory is mapped to a direct subdirectory of / within the container. +# (By present convention that subdirectory is: /prc) The test program to be run is given by its host path, and +# the internal (to the container) path will be computed. + +# The "-L" or leak option may be given as an instruction not to kill or remove the container after the test run. +# A sourced header for this script, 'test_script_parameters', contains configuration for each script that will +# be run under its control. + +IRODS_CONTROL_PATH="" +KILL_TEST_CONTAINER=1 +RUN_AS_USER="" +ECHO_CONTAINER="" +REMOVE_OPTION="--rm" +EXPLICIT_WORKDIR="" +INTERACTIVE_OPTION="" +VERBOSITY=0 + +while [[ $1 = -* ]]; do + if [ "$1" = -i ]; then + INTERACTIVE_OPTION="-it" + shift + fi + if [ "$1" = -V ]; then + VERBOSITY=1 + shift + fi + if [ "$1" = -c ]; then + ECHO_CONTAINER=1 + shift + fi + if [ "$1" = -L ]; then + KILL_TEST_CONTAINER=0 + shift + fi + if [ "$1" = -p ]; then + IRODS_CONTROL_PATH="$2" + shift 2 + fi + if [ "$1" = -u ]; then + RUN_AS_USER="$2" + shift 2 + fi + if [ "$1" = -r ]; then + REMOVE_OPTION="$2" + shift 2 + fi + if [ "$1" = -w ]; then + EXPLICIT_WORKDIR="$2" + shift 2 + fi +done + +if [ "$1" = "" ]; then + echo >&2 "Usage: $0 [options] /path/to/script" + echo >&2 "With options: [-L] to leak, [-u username] to run as non-root user" + exit 1 +fi + +DIR=$(dirname "$0") +. "$DIR"/test_script_parameters + +testscript=${1} +shift + +testscript_basename=$(basename "$testscript") +arglist=${wrapper_arglist[$testscript_basename]:-$*} # arglist dominated by symbolic link name if any + +if [ -L "$testscript" ]; then + testscript=$(realpath "$testscript") + testscript_basename=$(basename "$testscript") +fi + +original_testscript_abspath=$(realpath "$testscript") + +wrapped=${wrappers["$testscript_basename"]} + +if [ -n "$wrapped" ]; then + # wrapped is assumed to contain a leading path element relative to the referencing script's containing directory + testscript="$(dirname "$testscript")/$wrapped" + testscript_basename=$(basename "$testscript") +fi + +testscript_abspath=$(realpath "$testscript") + +cd "$DIR" + +image=${images[$testscript_basename]} + +if [ -z "$RUN_AS_USER" ]; then + RUN_AS_USER=${user[$testscript_basename]} +fi + +# Tests are run as testuser by default +: ${RUN_AS_USER:='testuser'} + +WORKDIR="" +if [ -n "$EXPLICIT_WORKDIR" ]; then + WORKDIR="$EXPLICIT_WORKDIR" +else + WORKDIR=${workdirs[$RUN_AS_USER]} +fi + +reporoot=$(./print_repo_root_location) +ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to "$reporoot" "$original_testscript_abspath") + +echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" +INNER_MOUNT=/prc + +: ${DOCKER:=docker} + +# Start the container. +echo image="[$image]" +CONTAINER=$($DOCKER run -d -v "$reporoot:$INNER_MOUNT:ro" $INTERACTIVE_OPTION $REMOVE_OPTION \ + -e "IRODS_CONTROL_PATH=$IRODS_CONTROL_PATH" $image) + +# Wait for iRODS and database to start up. +TIME0=$(date +%s) +while :; do + [ $(date +%s) -gt $((TIME0 + 30)) ] && { echo >&2 "Waited too long for DB and iRODS to start"; exit 124; } + sleep 1 + $DOCKER exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null + [ $? -ne 0 ] && { echo -n . >&2; continue; } + break +done + +if [ $VERBOSITY -gt 0 ]; then + echo $'\n'"==> Running script [$testscript_abspath]" + echo "in container [$CONTAINER]" + echo "with these *_VERSION variables in environment: " + $DOCKER exec $CONTAINER bash -c 'env|grep _VERSION' | sed $'s/^/\t/' +fi + +$DOCKER exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \ + ${WORKDIR:+"-w$WORKDIR"} \ + -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \ + -e "IRODS_CONTROL_PATH=$IRODS_CONTROL_PATH" \ + $INTERACTIVE_OPTION $CONTAINER \ + "$INNER_MOUNT/$(realpath --relative-to "$reporoot" "$testscript_abspath")" \ + $arglist +STATUS=$? + +if [ $((0+KILL_TEST_CONTAINER)) -ne 0 ]; then + echo >&2 'Killed:' $($DOCKER stop --timeout=0 $CONTAINER) +fi + +[ -n "$ECHO_CONTAINER" ] && echo $CONTAINER +exit $STATUS diff --git a/irods/test/harness/install.sh b/irods/test/harness/install.sh new file mode 100755 index 000000000..9a43a5dec --- /dev/null +++ b/irods/test/harness/install.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +# A script to manage the main steps in installing an iRODS server as well as all necessary support. (Dependencies, +# catalog database, etc.) + +IRODS_HOME=/var/lib/irods +DEV_HOME="$HOME" +: ${DEV_REPOS:="$DEV_HOME/github"} + +add_package_repo() +{ + echo >&2 "... installing package repo" + sudo apt update + sudo apt install -y lsb-release apt-transport-https gnupg2 + wget -qO - https://packages.irods.org/irods-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import \ + && \ + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods.list + + sudo apt update +} + +# Expand a spec of the leading version tuple eg. 4.3.4 out to the full name of +# the most recent matching version of the package + +# Report the latest version spec (including OS) that matches the env var IRODS_PACKAGE_VERSION (eg. "5.0.2" -> "5.0.2-0~jammy) + +irods_package_vsn() { + apt list -a irods-server 2>/dev/null|awk '{print $2}'|grep '\w'|sort|\ + grep "$(perl -e 'print quotemeta($ARGV[0])' "$IRODS_PACKAGE_VERSION")"|tail -1 +} + +# Report the version number of the installed iRODS server if any. + +irods_vsn() { + local V=$(dpkg -l irods-server 2>/dev/null|grep '^ii\s'|awk '{print $3}') + echo "${V}" +} + +while [[ "$1" = -* ]]; do + ARG="$1" + shift + case $ARG in + --i=* | --irods=* |\ + --irods-version=*) IRODS_PACKAGE_VERSION=${ARG#*=};; + --w=* | --with=* | --with-options=* ) withopts=${ARG#*=} ;; + esac +done + + +run_phase() { + + local PHASE=$1 + local with_opts=" $2 " + + case "$PHASE" in + + 0) + + if [[ $with_opts = *\ initialize\ * ]]; then + apt-get -y update + apt-get install -y apt-transport-https wget lsb-release sudo jq + fi + + if [[ $with_opts = *\ sudo-without-pw\ * ]]; then + if [ $(id -u) = 0 -a "${USER:-root}" = root ] ; then + echo >&2 "root authorization for 'sudo' is automatic - no /etc/sudoers modification needed" + else + if [ -f "/etc/sudoers" ]; then + # add a line with our USER name to /etc/sudoers if not already there + sudo su -c "sed -n '/^\s*[^#]/p' /etc/sudoers | grep '^$USER\s*ALL=(ALL)\s*NOPASSWD:\s*ALL\s*$' >/dev/null" || \ + sudo su -c "echo '$USER ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers" + else + echo >&2 "WARNING - Could not modify sudoers files" + fi + fi # not root + fi # with-opts + + #------ (needed for both package install and build from source) + + if [[ $with_opts = *\ install-essential-packages\ * ]]; then + + if ! dpkg -l tzdata >/dev/null 2>&1 ; then + sudo su - root -c \ + "env DEBIAN_FRONTEND=noninteractive bash -c 'apt-get install -y tzdata'" + fi + sudo apt-get update + sudo apt-get install -y software-properties-common postgresql + sudo apt-get update && \ + sudo apt-get install -y libfuse2 unixodbc rsyslog + fi + + + if [[ $with_opts = *\ add-package-repo\ * ]]; then + add_package_repo -f + fi + + + if [[ $with_opts = *\ create-db\ * ]]; then + sudo su - postgres -c " + { dropdb --if-exists ICAT + dropuser --if-exists irods ; } >/dev/null 2>&1" + sudo su - postgres -c "psql <<\\ +________ + CREATE DATABASE \"ICAT\"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE \"ICAT\" to irods; +________" + echo >&2 "-- status of create-db = $? -- " + fi + ;; + + 4) + IRODS_TO_INSTALL=$(irods_package_vsn) + sudo apt install -y irods-{dev,runtime}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + if [[ $with_opts != *\ basic\ * ]]; then + sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + fi + ;; + + 5) + if [ ! $(irods_vsn) '<' "4.3" ]; then + PYTHON=python3 + else + PYTHON=python2 + fi + sudo $PYTHON /var/lib/irods/scripts/setup_irods.py < /var/lib/irods/packaging/localhost_setup_postgres.input + ;; + + *) echo >&2 "unrecognized phase: '$PHASE'." ; QUIT=1 ;; + esac + return $? +} + +#-------------------------- main + +QUIT=0 +while [ $# -gt 0 ] ; do + ARG=$1 ; shift + NOP="" ; run_phase $ARG " $withopts "; sts=$? + [ $QUIT != 0 ] && break + [ -n "$NOP" ] && continue + echo -n "== $ARG == " + if [ $sts -eq 0 ]; then + echo Y >&2 + else + [ $quit_on_phase_err ] && { echo >&2 "N - quitting"; exit 1; } + echo N >&2 + fi +done diff --git a/irods/test/harness/install_python_rule_engine b/irods/test/harness/install_python_rule_engine new file mode 100755 index 000000000..036273fa1 --- /dev/null +++ b/irods/test/harness/install_python_rule_engine @@ -0,0 +1,23 @@ +#!/bin/bash + +python_rule_plugin_package_spec() { + local PKG=irods-rule-engine-plugin-python + local search_str="+$IRODS_PACKAGE_VERSION" + if [ "$1" = "all" ]; then + search_str="" + fi + local VERSIONS=$(apt list -a $PKG 2>/dev/null|\ + awk '{print $2}'|\ + grep '\w'|\ + grep "$( + perl -e 'print quotemeta($ARGV[0])' "$search_str")" + ) + local LATEST_VERSION=$(sort -V <<<"$VERSIONS" | tail -1) + if [ "$1" = "latest" ]; then + echo "$PKG=$LATEST_VERSION" + else + echo "$VERSIONS" + fi +} + +apt install -y "$(python_rule_plugin_package_spec latest)" diff --git a/irods/test/harness/irods_version_greater_or_equal_to b/irods/test/harness/irods_version_greater_or_equal_to new file mode 100755 index 000000000..a5375b144 --- /dev/null +++ b/irods/test/harness/irods_version_greater_or_equal_to @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import ast +import getopt +import glob +import json +import os +import sys + +def version_to_tuple(dot_str): + return tuple(int(_) for _ in dot_str.split('.')) + +# Fail unless iRODS version is greater than or equal to a version string given on the command line. +# With -e, we determine the installed iRODS version from the specified environment variable. +# Else use ~irods/version.json + +if __name__ == '__main__': + opts,arg = getopt.getopt (sys.argv[1:],'e:',['use_env_var=']) + + env_var = '' + + for opt,val in opts: + if opt in {'-e','--use_env_var'}: + env_var = val + + pattern = os.path.join(os.path.expanduser('~irods'), "*.json") + + # get full path of version.json + + if env_var: + version_to_test = os.environ[env_var] + else: + version_files = list( + filter( + (lambda name: name.lower().endswith('version.json')), + glob.glob(pattern) + ) + ) + # Load JSON struct containing iRODS version. + j = json.load(open(version_files[0])) + version_to_test = j["irods_version"] + + if version_to_tuple(version_to_test) < version_to_tuple(arg[0]): + exit(1) diff --git a/irods/test/harness/manage_irods5_procs b/irods/test/harness/manage_irods5_procs new file mode 100755 index 000000000..3327cb9be --- /dev/null +++ b/irods/test/harness/manage_irods5_procs @@ -0,0 +1,57 @@ +if [ `id -un` = irods ]; then + LAUNCH='bash -c' +else + LAUNCH='sudo su - irods -c' +fi + +STDOUT="" +PID="" + +start() { + if [ -z "$STDOUT" ] ; then + $LAUNCH 'irodsServer -d -p /tmp/irods.pid' + else + $LAUNCH 'irodsServer --stdout -p /tmp/irods.pid >/tmp/irods.log &' + fi +} + +rm_pid_file() { + if [ -z "$PID" ]; then + PID=$($LAUNCH 'cat /tmp/irods.pid') + fi + $LAUNCH 'rm -f /tmp/irods.pid >/dev/null 2>&1' +} + +stop() { + $LAUNCH 'kill -QUIT $(cat /tmp/irods.pid)' + rm_pid_file +} + +wait() { + $LAUNCH " + [ -z '$PID' ] && { echo >&2 'nothing to wait for.' ; exit 2; } + while ps -eo pid |grep $PID >/dev/null 2>&1; do sleep 1; done;" +} + +# ----------------------------------- +while [ -n "$1" ]; do + if [ "$1" = "stdout" ]; then + STDOUT=1 + elif [ "$1" = "start" ]; then + start + elif [ "$1" = "rescan-config" ]; then + $LAUNCH 'pkill -HUP irodsServer' + elif [ "$1" = "status" ]; then + pgrep -afl "irods(Delay|Agent|Server)" + elif [ "$1" = "stop" ]; then + stop + elif [ "$1" = "restart" ]; then + stop && start + elif [ "$1" = "wait" ]; then + wait + else + echo >&2 "usage: $0 [start|status|stop]" + exit 2 + fi + shift +done diff --git a/irods/test/harness/most_recent_python.sh b/irods/test/harness/most_recent_python.sh new file mode 100755 index 000000000..278f3ce7c --- /dev/null +++ b/irods/test/harness/most_recent_python.sh @@ -0,0 +1,25 @@ +#!/bin/bash +usage() { + echo >&2 "Usage: + $0 major.minor" + echo >&2 "Output: + prints full latest python version inclusive of the patch level." + exit 2 +} +MAJOR_MINOR=$1 +if [ -z "${MAJOR_MINOR}" ]; then # allow blank specification: most recent overall + MAJOR_MINOR='[0-9]\+\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ ^[0-9]+$ ]]; then # allow single integer, eg. 3 for most recent 3.y.z + MAJOR_MINOR+='\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ [0-9]+\.[0-9]+ ]]; then # allow x.y form, will yield output of most recent x.y.z + MAJOR_MINOR=$(sed 's/\./\\./'<<<"${MAJOR_MINOR}") # insert backslash in front of "." +elif ! [[ $MAJOR_MINOR =~ [0-9]+\\?.[0-9]+ ]]; then + usage +fi + +url='https://www.python.org/ftp/python/' + +# Fetch the directory listing, extract version numbers, sort them to find the largest numerically. +curl --silent "$url"|\ +sed -n 's!.*href="\('"${MAJOR_MINOR}"'\.[0-9]\+\)/".*!\1!p'|sort -rV|\ +head -n 1 diff --git a/irods/test/harness/print_repo_root_location b/irods/test/harness/print_repo_root_location new file mode 100755 index 000000000..adcf3fc73 --- /dev/null +++ b/irods/test/harness/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../../.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/irods/test/harness/setup_python_rule_engine b/irods/test/harness/setup_python_rule_engine new file mode 100755 index 000000000..d975ef7b4 --- /dev/null +++ b/irods/test/harness/setup_python_rule_engine @@ -0,0 +1,92 @@ +#!/bin/bash + +# This script should be run as the service account user. + +wait="" +if [ $1 = --wait ]; then + wait=1 + shift +fi + +DIR=$(dirname "$0") + +server_ctl() { + # This script takes one argument. + # Valid arguments: start, stop, or restart. The appropriate action is then taken for the resident iRODS server. + if "$DIR"/irods_version_greater_or_equal_to --use_env_var=IRODS_PACKAGE_VERSION 5.0; then + # Make our ps-based script wait for process shutdown like 'irodsctl stop' does. + W="" + if [ "$1" = stop ]; then + W=wait + fi + "$IRODS_CONTROL_PATH"/manage_irods5_procs $1 $W + else + ~/irodsctl $1 + fi +} + +jq_process_in_place() { + local filename=$1 + shift + local basenm=$(basename "$filename") + local tempname=/tmp/.$$.$basenm + + jq "$@" <"$filename" >"$tempname" && \ + cp "$tempname" "$filename" + STATUS=$? + rm -f "$tempname" + [ $STATUS = 0 ] || echo "**** jq process error" >&2 +} + +# -- Main part of script -- + +server_ctl stop + +jq_process_in_place /etc/irods/server_config.json \ + '.plugin_configuration.rule_engines[1:1]=[ { "instance_name": "irods_rule_engine_plugin-python-instance", + "plugin_name": "irods_rule_engine_plugin-python", + "plugin_specific_configuration": {} + } + ]' +echo ' +defined_in_both { + writeLine("stdout", "native rule") +} + +generic_failing_rule { + fail +} + +failing_with_message { + failmsg(-2, "error with code of minus 2") +} + +' >> /etc/irods/core.re + +echo ' +def defined_in_both(rule_args,callback,rei): + callback.writeLine("stdout", "python rule") + +def generic_failing_rule(*_): + raise RuntimeError + +def failing_with_message_py(rule_args,callback,rei): + callback.failing_with_message() + +' > /etc/irods/core.py + +server_ctl start + +# Wait until 'irule -a' shows Python Rule Engine Plugin among the choices +if [ -n "$wait" ]; then + times=0 + OUTFILE=/tmp/irule_output.stderr + while :; do + irule -a 2>/dev/null| grep irods_rule_engine_plugin-python-instance >/dev/null + [ ${PIPESTATUS[1]} -eq 0 ] && break + sleep 1 + if [ $((++times)) -ge 10 ]; then + echo >&2 "Failed to configure Python rule engine."; exit 2; + fi + done +fi diff --git a/irods/test/harness/start_postgresql_and_irods.sh b/irods/test/harness/start_postgresql_and_irods.sh new file mode 100755 index 000000000..ebe7b540d --- /dev/null +++ b/irods/test/harness/start_postgresql_and_irods.sh @@ -0,0 +1,34 @@ +#!/bin/bash +service postgresql start +x=${DB_WAIT_SEC:-20} +while [ $x -ge 0 ] && { ! $SUDO su - postgres -c "psql -c '\l' >/dev/null 2>&1" || x=""; } +do + [ -z "$x" ] && break + echo >&2 "$((x--)) secs til database timeout"; sleep 1 +done +[ -z "$x" ] || { echo >&2 "Error -- database didn't start" ; exit 1; } +VERSION_file=$(ls /var/lib/irods/{VERSION,version}.json.dist 2>/dev/null) +if ! id -u irods >/dev/null 2>&1 ; then + /install.sh --w=create-db 0 + /install.sh 5 +fi +IRODS_VSN=$(jq -r '.irods_version' $VERSION_file) +IRODS_VSN_MAJOR=${IRODS_VSN//.*/} +if [ "$IRODS_VSN_MAJOR" -lt 5 ]; then + su - irods -c '~/irodsctl restart' +else + "$IRODS_CONTROL_PATH"/manage_irods5_procs stdout start +fi +IRODS_WAIT_SEC=20 +x=$IRODS_WAIT_SEC +SLEEPTIME="" +while [ $((x--)) -gt 0 ]; do + sleep $((SLEEPTIME+0)) + pgrep irodsServer + STATUS=$? + [ $STATUS -eq 0 ] && break + SLEEPTIME=1 +done +echo "($STATUS)" >/tmp/irods_status +[ $STATUS -eq 0 ] || exit 125 +tail -f /dev/null diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters new file mode 100644 index 000000000..fe2beca54 --- /dev/null +++ b/irods/test/harness/test_script_parameters @@ -0,0 +1,44 @@ +# keys for Arglist refer to argument given, which could be a symlink. + +declare -A wrapper_arglist=( + [login_auth_test_must_run_manually.py]="-v TestLogins" + [login_auth_test_1.py]="-v TestAnonymousUser TestMiscellaneous" + [login_auth_test_2.py]="-v TestWithSSL" +) + +# keys for Wrapper refer to argument after resolution of any symlinks + +declare -A wrappers=( + [login_auth_test_must_run_manually.py]=./login_auth_test.sh + [PRC_issue_362.bats]=./login_auth_test.sh + [test001_pam_password_expiration.bats]=../login_auth_test.sh + [test002_write_native_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test003_write_pam_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test004_prc_pam_password_internal_secrets_file_generation.bats]=../login_auth_test.sh + [test005_test_special_characters_in_pam_passwords.bats]=../login_auth_test.sh + [test006_connection_timeout_on_ssl_socket.bats]=../login_auth_test.sh + [test007_pam_features_in_new_auth_framework.bats]=../login_auth_test.sh + [test008_prc_write_irodsA_utility_in_native_mode.bats]=../login_auth_test.sh + [test009_test_special_characters_in_pam_passwords_auth_framework.bats]=../login_auth_test.sh + [test010_issue_362_rogue_chars_in_pam_password.bats]=../login_auth_test.sh +) + +# keys for Image and User refer to the basename after resolution to a wrapper if one is used + +declare -A images=( + [login_auth_test.sh]=compile-specific-python + [login_auth_test_must_run_manually.py]=ssl-and-pam + [run_suite_locally.sh]=compile-specific-python +) + +declare -A user=( + [run_suite_locally.sh]=root +) + +# keys for WorkDir refer to user + +declare -A workdirs=( + [testuser]=/home/testuser + [irods]=/var/lib/irods + [root]=/ +) diff --git a/irods/test/login_auth_test.sh b/irods/test/login_auth_test.sh new file mode 100755 index 000000000..9ef4c3b15 --- /dev/null +++ b/irods/test/login_auth_test.sh @@ -0,0 +1,77 @@ +#!/bin/bash +. "$(dirname "$0")/scripts/test_support_functions" +. "$(dirname "$0")/scripts/update_json_for_test" + +IRODS_SERVER_CONFIG=/etc/irods/server_config.json +IRODS_SERVICE_ACCOUNT_ENV_FILE=~irods/.irods/irods_environment.json +LOCAL_ACCOUNT_ENV_FILE=~/.irods/irods_environment.json + +cannot_iinit='' +tries=8 +while true; do + iinit_as_rods >/dev/null 2>&1 && break + [ $((--tries)) -le 0 ] && { cannot_iinit=1; break; } + sleep 5 +done +[ -n "$cannot_iinit" ] && { echo >&2 "Could not iinit as rods."; exit 2; } + +setup_preconnect_preference DONT_CARE + +add_irods_to_system_pam_configuration + + +# set up /etc/irods/ssl directory and files +set_up_ssl sudo + +sudo useradd -ms/bin/bash alissa +sudo chpasswd <<<"alissa:test123" + +update_json_file $IRODS_SERVICE_ACCOUNT_ENV_FILE \ + "$(newcontent $IRODS_SERVICE_ACCOUNT_ENV_FILE ssl_keys)" + +# This is mostly so we can call python3 as just "python" +activate_virtual_env_with_prc_installed >/dev/null 2>&1 || { echo >&2 "couldn't set up virtual environment"; exit 1; } + +server_hup= +if irods_server_version ge 5.0.0; then + server_hup="y" + update_json_file $IRODS_SERVER_CONFIG \ + "$(newcontent $IRODS_SERVER_CONFIG tls_server_items tls_client_items)" + + sudo su - irods -c "$IRODS_CONTROL_PATH/manage_irods5_procs rescan-config" +fi + +# Configure clients with admin user + TLS + +update_json_file $LOCAL_ACCOUNT_ENV_FILE \ + "$(newcontent $LOCAL_ACCOUNT_ENV_FILE ssl_keys encrypt_keys)" + +if [ "$server_hup" = y ]; then + # wait for server to be ready after configuration reload + while true; do + sleep 2 + if ils >/dev/null 2>&1; then + break + else + # Allow ~16 secs of total wait time. + [ $((++server_check)) -gt 8 ] && { + echo >&2 "Timed out on server reload"; exit 3; } + fi + done +fi + +if [ -n "$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" ]; then + original_script="/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" + + # Run tests. + if [ -x "$original_script" ]; then + command "$original_script" $* + elif [[ $original_script =~ \.py$ ]]; then + python "$original_script" $* + elif [[ $original_script =~ \.bats$ ]]; then + bats "$original_script" + else + echo >&2 "I don't know how to run this: original_script=[$original_script]" + fi + +fi diff --git a/irods/test/login_auth_test_1.py b/irods/test/login_auth_test_1.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_1.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_2.py b/irods/test/login_auth_test_2.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_2.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_must_run_manually.py b/irods/test/login_auth_test_must_run_manually.py index 4303e47c1..a6caadccc 100644 --- a/irods/test/login_auth_test_must_run_manually.py +++ b/irods/test/login_auth_test_must_run_manually.py @@ -207,6 +207,9 @@ def create_env_dirs(self): authentication_scheme=lookup["AUTH"], password=lookup["PASSWORD"], port=1247, + **( + {**SERVER_ENV_SSL_SETTINGS, **CLIENT_OPTIONS_FOR_SSL} if self.admin.server_version >= (5,) else {} + ) ) try: pam_hashes = ses.pam_pw_negotiated @@ -295,6 +298,7 @@ def _setup_rodsuser_and_optional_pw(self, name, make_irods_pw=False): def tst0( self, ssl_opt, auth_opt, env_opt, name=TEST_RODS_USER, make_irods_pw=False ): + session = None _auth_opt = auth_opt if auth_opt in ("pam", "pam_password"): auth_opt = self.PAM_SCHEME_STRING @@ -377,7 +381,10 @@ def tst0( ) print("---") - return session + if session: + session.cleanup() + return session + # == test defaulting to 'native' @@ -413,14 +420,16 @@ def test_5(self): self.tst0(ssl_opt=True, auth_opt="pam", env_opt=False) def test_6(self): + if self.admin.server_version >= (5,): + self.skipTest("iRODS 5 does not permit sending the raw PAM password on an unencrypted connection.") try: - ses = self.tst0(ssl_opt=False, auth_opt="pam", env_opt=False) + session = self.tst0(ssl_opt=False, auth_opt="pam", env_opt=False) except PlainTextPAMPasswordError: pass else: # -- no exception raised (this is expected behavior in 4.3+ with the new authentication framework, # but for 4.2 and previous, we expect the PlainTextPAMPasswordError to be raised. - if ses.server_version_without_auth() < (4, 3): + if session.server_version_without_auth() < (4, 3): self.fail("PlainTextPAMPasswordError should have been raised") def test_7(self): @@ -534,7 +543,7 @@ def test_nonanonymous_login_without_auth_file_fails__290(self): s.users.get("bob") os.unlink(bob_auth) # -- Check that we raise an appropriate exception pointing to the missing auth file path -- - with self.assertRaisesRegexp(NonAnonymousLoginWithoutPassword, bob_auth): + with self.assertRaisesRegex(NonAnonymousLoginWithoutPassword, bob_auth): with helpers.make_session(**login_options) as s: s.users.get("bob") finally: diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index aa33dae26..1a0d01bf4 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -683,16 +683,16 @@ def test_AVUs_populated_improperly_with_empties_or_nonstrings_fail_identically__ def test_nonstring_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", 0) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.add(*args) def test_empty_string_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", "") - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.add(*args) @unittest.skipUnless( @@ -724,10 +724,11 @@ def test_that_all_column_mappings_are_uniquely_and_properly_defined__issue_643( prepend_col_prefix_if_needed = lambda s: ( "COL_" + s if not s.startswith("COL_") else s ) + current_server_version = self.sess.server_version prc_column_defs = sorted( [ (prepend_col_prefix_if_needed(i[1].icat_key), i[1].icat_id) - for i in ModelBase.column_items + for i in ModelBase.column_items if current_server_version >= i[1].min_version ] ) diff --git a/irods/test/pam.bats/funcs b/irods/test/pam.bats/funcs deleted file mode 100644 index 30539a03a..000000000 --- a/irods/test/pam.bats/funcs +++ /dev/null @@ -1,108 +0,0 @@ -dot_to_space() { - sed 's/\./ /g'<<<"$1" -} - -CLEANUP=$':\n' - -GT() { (return 1); echo $?; } -LT() { (return -1); echo $?; } -EQ() { (return 0); echo $?; } - -compare_int_tuple() { - local x=($1) y=($2) - local lx=${#x[@]} ly=${#y[@]} - local i maxlen=$((lx > ly ? lx : ly)) - for ((i=0;i ~/.irods/irods_environment.json - iinit <<<"$1" 2>/dev/tty -} - -_end_pam_environment_and_password() { - rm -fr ~/.irods - mv ~/.irods.$$ ~/.irods -} - -setup_pam_login_for_alice() { - sudo useradd alice --create-home - local PASSWD=${1:-test123} - sudo chpasswd <<<"alice:$PASSWD" - iadmin mkuser alice rodsuser - _begin_pam_environment_and_password "$PASSWD" -} - -finalize_pam_login_for_alice() { - _end_pam_environment_and_password - iadmin rmuser alice - sudo userdel alice --remove -} - -test_specific_cleanup() { - eval "$CLEANUP" -} diff --git a/irods/test/pam.bats/test001_pam_password_expiration.bats b/irods/test/pam.bats/test001_pam_password_expiration.bats deleted file mode 100644 index 3e29100ef..000000000 --- a/irods/test/pam.bats/test001_pam_password_expiration.bats +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bats - -. "$BATS_TEST_DIRNAME"/test_support_functions -PYTHON=python3 - -# Setup/prerequisites are same as for login_auth_test. -# Run as ubuntu user with sudo; python_irodsclient must be installed (in either ~/.local or a virtualenv) -# - -PASSWD=test123 - -setup() -{ - setup_pam_login_for_alice $PASSWD -} - -teardown() -{ - finalize_pam_login_for_alice - test_specific_cleanup -} - -@test f001 { - - # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS - # without an exception being raised. - - local SCRIPT=" -import irods.test.helpers as h -ses = h.make_session() -ses.collections.get(h.home_collection(ses)) -print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) -" - - # Test that the first run of the code in $SCRIPT is successful, i.e. normal authenticated operations are possible. - - local OUTPUT=$($PYTHON -c "$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - SET_CLEANUP=yes \ - with_change_auth_params_for_test password_min_time 4 \ - password_max_time 5 - - # Test that running the $SCRIPT raises an exception if the PAM password has expired. - - iinit <<<"$PASSWD" - HOME_COLLECTION=$(ipwd) - sleep 9 - OUTPUT=$($PYTHON -c "$SCRIPT" 2>&1 >/dev/null || true) - grep 'RuntimeError: Time To Live' <<<"$OUTPUT" - - # Test that the $SCRIPT, when run with proper settings, can successfully reset the password. - - with_change_auth_params_for_test password_max_time 3600 - - OUTPUT=$($PYTHON -c "import irods.client_configuration as cfg -cfg.legacy_auth.pam.password_for_auto_renew = '$PASSWD' -cfg.legacy_auth.pam.time_to_live_in_hours = 1 -cfg.legacy_auth.pam.store_password_to_environment = True -$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - # Test that iCommands can authenticate with the newly written .irodsA file - - iquest "%s" "select COLL_NAME where COLL_NAME like '%/home/alice%'"| grep "^$HOME_COLLECTION\$" -} diff --git a/irods/test/pam_interactive_test.py b/irods/test/pam_interactive_test_must_run_manually.py similarity index 100% rename from irods/test/pam_interactive_test.py rename to irods/test/pam_interactive_test_must_run_manually.py diff --git a/irods/test/rule_test.py b/irods/test/rule_test.py index be95302b7..39b718a26 100644 --- a/irods/test/rule_test.py +++ b/irods/test/rule_test.py @@ -20,7 +20,7 @@ RE_Plugins_installed_run_condition_args = ( os.environ.get("PYTHON_RULE_ENGINE_INSTALLED", "*").lower()[:1] == "y", - "Test depends on server having Python-REP installed beyond the default options", + "Test depends on server having Python-REP installed (set PYTHON_RULE_ENGINE_INSTALLED=yes in environment)." ) @@ -420,7 +420,7 @@ def test_rulefile_in_file_like_object_1__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r".*\[Hello world!\]") + self.assertRegex(lines[0], r".*\[Hello world!\]") def test_rulefile_in_file_like_object_2__336(self): @@ -442,8 +442,8 @@ def test_rulefile_in_file_like_object_2__336(self): r = Rule(self.sess, rule_file=io.BytesIO(rule_file_contents.encode("utf-8"))) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[STRING\]\[\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[\]") + self.assertRegex(lines[0], r"\[STRING\]\[\]") + self.assertRegex(lines[1], r"\[STRING\]\[\]") r = Rule( self.sess, @@ -452,8 +452,8 @@ def test_rulefile_in_file_like_object_2__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[INTEGER\]\[5\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[A String\]") + self.assertRegex(lines[0], r"\[INTEGER\]\[5\]") + self.assertRegex(lines[1], r"\[STRING\]\[A String\]") if __name__ == "__main__": diff --git a/irods/test/runner.py b/irods/test/runner.py index f9f9fa610..8782d6391 100644 --- a/irods/test/runner.py +++ b/irods/test/runner.py @@ -7,6 +7,7 @@ """ +import argparse import os import sys from unittest import TestLoader, TestSuite @@ -22,20 +23,74 @@ h.setFormatter(f) logger.addHandler(h) +parser = argparse.ArgumentParser() -# Load all tests in the current directory and run them +def abs_path(initial_dir, levels_up = 0): + directory = initial_dir + while levels_up > 0: + levels_up -= 1 + directory = os.path.join(directory,'..') + return os.path.abspath(directory) + +# Load all tests in the current directory and run them. if __name__ == "__main__": - # must set the path for the imported tests - sys.path.insert(0, os.path.abspath("../..")) + + # Get path to script directory for test import and/or discovery. + script_dir = os.path.abspath(os.path.dirname(sys.argv[0])) + + # Must set the path for the imported tests. + sys.path.insert(0, abs_path(script_dir, levels_up = 2)) + + parser.add_argument('--tests', '-t', + metavar='TESTS', + dest='tests', + nargs='+', + help='List of tests to run.') + + parser.add_argument('--environment_variable', '-e', + metavar='ENVIRONMENT_VARIABLE', + dest='env_var', + type=str, + help='Name of environment variable name to scan for in reason strings when filtering skipped test names to be output.') + + parser.add_argument('--output_tests_skipped', '-s', + metavar='SKIPPED_TESTS_OUTPUT_FILENAME', + dest='skipped_tests_output_filename', + type=str, + help='Name of a file into which to write names of skipped tests.') + + parser.add_argument('--tests_file', '-f', + metavar='TESTS_FILE', + dest='tests_file', + help='Name of a file containing a list of tests to run.') + + args = parser.parse_args() + + if args.tests_file: + if args.tests: + print ('Cannot specify both --tests and --tests_file', file = sys.stderr) + exit(2) + args.tests = filter(None,open(args.tests_file).read().split("\n")) loader = TestLoader() - suite = TestSuite( - loader.discover(start_dir=".", pattern="*_test.py", top_level_dir=".") - ) + + if args.tests: + suite = TestSuite(loader.loadTestsFromNames(args.tests)) + else: + suite = TestSuite(loader.discover(start_dir = script_dir, pattern = '*_test.py', top_level_dir = script_dir)) result = xmlrunner.XMLTestRunner( verbosity=2, output="/tmp/python-irodsclient/test-reports" ).run(suite) + + if args.skipped_tests_output_filename: + with open(args.skipped_tests_output_filename,'w') as skip_file: + do_output = (lambda reason: (args.env_var in reason) if args.env_var + else True) + for testinfo, reason in result.skipped: + if do_output(reason): + print(testinfo.test_id, file=skip_file) + if result.wasSuccessful(): sys.exit(0) diff --git a/irods/test/scripts/iinit.py b/irods/test/scripts/iinit.py new file mode 120000 index 000000000..cb4b84b21 --- /dev/null +++ b/irods/test/scripts/iinit.py @@ -0,0 +1 @@ +../../../docker-testing/iinit.py \ No newline at end of file diff --git a/irods/test/scripts/run_suite_locally.sh b/irods/test/scripts/run_suite_locally.sh new file mode 100755 index 000000000..da7828131 --- /dev/null +++ b/irods/test/scripts/run_suite_locally.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR=$(dirname "$0") +. "$SCRIPT_DIR"/test_support_functions + +report_environment_variables() { + echo "PRC under test with these environment variables active:" + python -c " +import os, sys +for name in ['IRODS_PACKAGE_VERSION','PYTHON_VERSION']: + value = os.environ.get(name) + print(f' {name}=[{value}]') +print(f'{sys.executable = }') +print(f'{sys.version = }') + " +} + +run_tests() { + setup_pyN + su - testuser -c " + set -e + source /pyN/bin/activate + pip install -e /prc.rw[tests] + cd /prc.rw/irods/test + python /prc.rw/docker-testing/iinit.py \ + host localhost \ + port 1247 \ + user rods \ + zone tempZone \ + password rods + $(declare -f report_environment_variables) + report_environment_variables + python runner.py --output_tests_skipped /tmp/skipped.txt -e PYTHON_RULE_ENGINE_INSTALLED + " + + # Install PREP (Python Rule Engine Plugin). + ( + set -e + cd "$SCRIPT_DIR/../harness" + apt update + ./install_python_rule_engine + su irods -c './setup_python_rule_engine --wait' + ) + + # Run PREP-dependent tests that were previously skipped. + su - testuser -c " + set -e + source /pyN/bin/activate + cd /prc.rw/irods/test + $(declare -f report_environment_variables) + report_environment_variables + env PYTHON_RULE_ENGINE_INSTALLED=yes python runner.py --tests_file /tmp/skipped.txt + " +} + +run_tests diff --git a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats index acf9c9594..8d84ddf82 100755 --- a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats +++ b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats @@ -43,9 +43,13 @@ except irods.client_init.irodsA_already_exists: [ -n "$CONTENTS1" -a "$CONTENTS1" = "$CONTENTS2" ] # Now delete the already existing irodsA and repeat without negating overwrite. + TIMESTAMP_0=$(stat -c%Y $auth_file) + sleep 2 $PYTHON -c "import irods.client_init; irods.client_init.write_pam_irodsA_file('$ALICES_NEW_PAM_PASSWD')" - CONTENTS3=$(cat $auth_file) - [ "$CONTENTS2" != "$CONTENTS3" ] + TIMESTAMP=$(stat -c%Y $auth_file) + + # Test only the timestamp of the new auth_file, not the content, since that is implicitly asserted by the next step. + [ $(($TIMESTAMP-TIMESTAMP_0)) -ge 1 ] # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS # without an exception being raised. diff --git a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats index 23aecd8ce..de9304461 100755 --- a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats +++ b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats @@ -48,6 +48,9 @@ print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) # Write another .irodsA prc_write_irodsA.py native <<<"rods" + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_client_server_policy"]="CS_NEG_REFUSE"' <$CLIENT_JSON >/tmp/client_json_test008.$$ + mv /tmp/client_json_test008.$$ $CLIENT_JSON # Verify new .irodsA for both iCommands and PRC use. ils >/tmp/stdout OUTPUT=$($PYTHON -c "$SCRIPT") diff --git a/irods/test/PRC_issue_362.bats b/irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats old mode 100644 new mode 100755 similarity index 64% rename from irods/test/PRC_issue_362.bats rename to irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats index c3a1ef80f..308b0af8e --- a/irods/test/PRC_issue_362.bats +++ b/irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats @@ -1,15 +1,26 @@ +#!/usr/bin/env bats + # The tests in this BATS module must be run as a (passwordless) sudo-enabled user. # It is also required that the python irodsclient be installed under irods' ~/.local environment. -. $BATS_TEST_DIRNAME/scripts/test_support_functions +. $BATS_TEST_DIRNAME/test_support_functions setup() { - - iinit_as_rods - - setup_pam_login_for_user "test123" alice - - cat >~/test_get_home_coll.py <<-EOF + [ -f /tmp/once ] || { + rm -fr ~/.irods + $BATS_TEST_DIRNAME/iinit.py host localhost \ + port 1247 \ + zone tempZone \ + user rods \ + password rods \ + ## Because iRODS 5+ negotiates for SSL automatically: + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_client_server_policy"]="CS_NEG_REFUSE"' >$CLIENT_JSON.$$ <$CLIENT_JSON + mv $CLIENT_JSON.$$ $CLIENT_JSON + + setup_pam_login_for_user "test123" alice + + cat >~/test_get_home_coll.py <<-EOF import irods.test.helpers as h ses = h.make_session() home_coll = h.home_collection(ses) @@ -17,11 +28,8 @@ setup() { and ses.pool.account._original_authentication_scheme.lower() in ('pam','pam_password') else 1) EOF -} - -teardown() { - iinit_as_rods - finalize_pam_login_for_user alice + } + touch /tmp/once } prc_test() diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions new file mode 100644 index 000000000..06fd0df48 --- /dev/null +++ b/irods/test/scripts/test_support_functions @@ -0,0 +1,242 @@ +# This is effectively a group of utility functions to be sourced from tests +# written in BATS or in straight Bash. Many or most have to do either with: +# +# 1. implementing common tasks, most often setup or configuration wrt iRODS, or +# 2. more primitive functions e.g. string manipulations or comparisons, etc. + +SCRIPTDIR=${BASH_SOURCE[0]} +up_from_script_dir() { + local x incr="" + for ((x=0;x<${1:-0};x++)); do incr+="/.."; done + realpath "$(dirname "$SCRIPTDIR")""$incr" +} + +# Sample usages: +# By user irods: set_up_ssl "" "-q" +# By sudo enabled user: set_up_ssl "sudo" "-q" +set_up_ssl() { + local SUDO=${1:-""} + local OPTS=${2:-""} + $SUDO su - irods -c "python3 $(up_from_script_dir 1)/setupssl.py $OPTS" +} + +# Clears out environment and resets to rodsadmin 'rods'. +# Meant mostly to allow initial steps by a rodsadminfor setting up tests. + +iinit_as_rods() { + rm -fr ~/.irods + iinit <<<$(hostname)$'\n1247\nrods\ntempZone\nrods' +} + +dot_to_space() { + sed 's/\./ /g'<<<"$1" +} + +CLEANUP=$':\n' + +GT() { (return 1); echo $?; } +LT() { (return -1); echo $?; } +EQ() { (return 0); echo $?; } + +compare_int_tuple() { + local x=($1) y=($2) + local lx=${#x[@]} ly=${#y[@]} + local i maxlen=$((lx > ly ? lx : ly)) + for ((i=0;i ~/.irods/irods_environment.json + + # TODO: check: it seems /dev/tty won't work if docker exec is not invoked with -t + if [ -n "$1" -a -z "$SKIP_IINIT_FOR_PASSWORD" ]; then + iinit <<<"$1" 2>/tmp/iinit_as_alice.log + fi +} + +_end_pam_environment_and_password() { + rm -fr ~/.irods + mv ~/.irods.$$ ~/.irods +} + +setup_pam_login_for_user() { + local user=${2:-alice} + sudo useradd $user --create-home + local PASSWD=${1:-test123} + sudo chpasswd <<<"$user:$PASSWD" + iadmin mkuser $user rodsuser + _begin_pam_environment_and_password "$PASSWD" $user +} + +setup_pam_login_for_alice() { + setup_pam_login_for_user "$1" alice +} + +finalize_pam_login_for_user() { + local USER=${1} + _end_pam_environment_and_password + iadmin rmuser "$USER" + sudo userdel "$USER" --remove +} + +finalize_pam_login_for_alice() { + finalize_pam_login_for_user alice +} + +test_specific_cleanup() { + eval "$CLEANUP" +} + +# PostgreSQL only +age_out_pam_password() { + # sets create_ts and modify_ts (timestamps) to older values, decreasing them by an amount of (offset + 1) where offset + # is the number of seconds for expiry_ts stored in the ICAT for the given user and password. In this way, we can + # artificially age out an existing pam password. + # Parameters: + # $1 - The username + # $2 - (optional) override the amount used for offsetting the create & modify timestamps. + local id=$(iquest %s "select USER_ID where USER_NAME = '$1'") + local offset=$(sudo su - postgres -c "psql -t ICAT -c 'select pass_expiry_ts from r_user_password where user_id = $id'") + local mtime=$(sudo su - postgres -c "psql -t ICAT -c 'select modify_ts from r_user_password where user_id = $id'") + mtime=$(sed 's/^\s*0//' <<<"$mtime") + [ -n "$2" ] && offset="$2" + ((offset+=1)) + local new_time=$((mtime - offset)) + sudo su - postgres -c "psql ICAT -c 'update r_user_password set create_ts=$new_time, modify_ts=$new_time where user_id=$id'" +} + +call_irodsctl() { + local arg=${1:-restart} + sudo su - irods -c "./irodsctl $arg" +} + +add_irods_to_system_pam_configuration() { + local tempfile=/tmp/irods-pam-config.$$ + cat <<-EOF >$tempfile + auth required pam_env.so + auth sufficient pam_unix.so + auth requisite pam_succeed_if.so uid >= 500 quiet + auth required pam_deny.so + EOF + sudo chown root.root $tempfile + sudo mv $tempfile /etc/pam.d/irods +} + +setup_preconnect_preference() { + sudo su irods -c "sed -i.orig 's/\(^\s*acPreConnect.*CS_NEG\)\([A-Z_]*\)/\1_$1/' /etc/irods/core.re" +} + +setup_pyN() { + if [ ! -d /pyN ]; then + mkdir /pyN ; chown testuser /pyN + su - testuser -c "/root/python/bin/python3 -m virtualenv /pyN" + cp -r /prc{,.rw} + chown -R testuser /prc.rw + fi +} + +# requires image to descend from bats-python3 +activate_virtual_env_with_prc_installed() +{ + local py_venv=${1:-pyN} + # install python client. We use a recursive copy of /prc so bdist doesn't try to build from a readonly mount + [ "$py_venv" = pyN ] && sudo bash -c "$(declare -f setup_pyN); setup_pyN" && \ + sudo su - -c "source /${py_venv}/bin/activate && cp -rp /prc /prc-copy && \ + pip install '/prc-copy[tests]' && sudo rm -fr /prc-copy" && \ + source /${py_venv}/bin/activate && \ + echo "---> Python virtual environment activated. Interpreter Version is: $(python -V)" && \ + python -c 'import irods' || { echo >&2 "ERROR: python-irodsclient install failed."; false; } +} + +mtime_and_content() +{ + stat -c%y "$1" + cat "$1" +} + +irods_server_version() { + python -c "import irods.helpers as h +import operator,sys +if len(sys.argv) == 1: + (comparison,relto)=('','') +elif len(sys.argv) == 3: + (comparison,relto)=sys.argv[1:3] +fm_tuple = lambda tup: '.'.join(str(_) for _ in tup) +to_tuple = lambda vstr: tuple(int(_) for _ in vstr.split('.')) +svt = h.make_session().server_version_without_auth() +if relto: + exit(0 if vars(operator)[comparison](svt,to_tuple(relto)) else 1) +print(fm_tuple(svt)) +" $1 $2 +} diff --git a/irods/test/scripts/update_json_for_test b/irods/test/scripts/update_json_for_test new file mode 100644 index 000000000..3372fd48e --- /dev/null +++ b/irods/test/scripts/update_json_for_test @@ -0,0 +1,69 @@ +#!/bin/bash + +declare -A tls_server_items=( + [tls_server]='{"certificate_chain_file":"/etc/irods/ssl/irods.crt", + "certificate_key_file":"/etc/irods/ssl/irods.key", + "dh_params_file":"/etc/irods/ssl/dhparams.pem"}' +) + +declare -A tls_client_items=( + [tls_client]='{"ca_certificate_file":"/etc/irods/ssl/irods.crt", + "ca_certificate_path":"/etc/ssl/certs", + "verify_server":"cert"}' +) + +declare -A ssl_keys=( + [irods_client_server_negotiation]='"request_server_negotiation"' + [irods_client_server_policy]='"CS_NEG_REQUIRE"' + [irods_ssl_ca_certificate_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_chain_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_key_file]='"/etc/irods/ssl/irods.key"' + [irods_ssl_dh_params_file]='"/etc/irods/ssl/dhparams.pem"' + [irods_ssl_verify_server]='"cert"' +) + +declare -A pam_keys=( + [irods_authentication_scheme]="\"$(pam_auth_string)\"" +) + +declare -A encrypt_keys=( + [irods_encryption_key_size]=16 + [irods_encryption_salt_size]=8 + [irods_encryption_num_hash_rounds]=16 + [irods_encryption_algorithm]='"AES-256-CBC"' +) + +declare -A RESTORE_FILES=() + +update_json_file() { + local file=$1 content=$2 + local bn=$(basename "$file") + local orig=/tmp/$bn.orig.$$ + local newfile=/tmp/$bn.new.$$ + echo "$content" >"$newfile" + sudo chmod --reference "$file" "$newfile" + sudo chown --reference "$file" "$newfile" + { sudo mv "$file" "$orig" && sudo mv "$newfile" "$file"; } || return 1 + RESTORE_FILES["$file"]="$orig" +} + +restore_json_files() { + local kk + for kk in ${!RESTORE_FILES[@]};do + sudo mv -f "${RESTORE_FILES["$kk"]}" "$kk" + done +} + +newcontent () { + local file=$1 + shift + local j=$(sudo cat "$file") + while [ $# -gt 0 ]; do + eval ' + for kk in ${!'$1'[@]}; do + j=$(jq ".$kk=${'$1'[$kk]}" <<<"$j") + done' + shift + done + echo "$j" +} diff --git a/irods/test/setupssl.py b/irods/test/setupssl.py index 3d3c20205..d14e682c0 100755 --- a/irods/test/setupssl.py +++ b/irods/test/setupssl.py @@ -60,7 +60,7 @@ def create_ssl_dir( # https://www.openssl.org/docs/man1.0.2/man1/dhparam.html#:~:text=DH%20parameter%20generation%20with%20the,that%20may%20be%20possible%20otherwise. if use_strong_primes_for_dh_generation: dhparam_generation_command = ( - "openssl dhparam -2 -out dhparams.pem" + "openssl dhparam -2 -out dhparams.pem 2048" ) else: dhparam_generation_command = (