[#82][#97][#95] Added test framework and some test cases

Note: This handles the first bullet points in 95 and 97.

.gitignore

@@ -9,3 +9,5 @@ testirodsmap
 gridmap_iRODS_callout.conf
 .cproject
 .project
+*.pyc
+*.bu

README.md

@@ -332,6 +332,11 @@ export irodsEnvFile=/path/to/.irodsEnv
 $DEST_BIN_DIR/testirodsmap "/C=XX/O=YYY/CN=Example User"
 ```
 
+Automated Testing
+-----------------
+
+See [Automated Testing README](tests/docker/README.md) for information about running the automated testing scripts.
+
 
 License
 ---------------------------------

tests/__init__.py

@@ -0,0 +1 @@
+# For now, this test suite requires the iCommands packages.

tests/docker/00-irods.conf

@@ -0,0 +1,8 @@
+$FileCreateMode 0644
+$DirCreateMode 0755
+$Umask 0000
+$template irods_format,"%msg%\n"
+:programname,startswith,"irodsServer" /var/log/irods/irods.log;irods_format
+& stop
+:programname,startswith,"irodsDelayServer" /var/log/irods/irods.log;irods_format
+& stop

tests/docker/README.md

@@ -0,0 +1,23 @@
+Test Framework
+==============
+
+This is a test framework for testing the irods_client_globus_connector.
+
+Running Tests
+-------------
+
+To run the tests first make sure you have the code that is to be tested checked out.
+
+Run the tests by executing the following in this directory:
+
+```
+docker-compose build
+docker-compose run globus
+```
+
+Test Limitations
+----------------
+
+Currently this only runs against a Globus server running Ubuntu 22.  This will be expanded in the future.
+
+The `python run_all_tests.py` command can be run against Globus installations in other operating systems but those will need to be set up manually.  Refer to [globus start script](start.globus.ubuntu22.sh) for an example.

tests/docker/db_commands.txt

@@ -0,0 +1,4 @@
+CREATE DATABASE "ICAT";
+CREATE USER irods WITH PASSWORD 'testpassword';
+GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods;
+\q

tests/docker/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3'
+
+services:
+    irods:
+        build:
+            context: . 
+            dockerfile: irods.ubuntu22.Dockerfile
+        healthcheck:
+            test: ils || exit 1
+            start_period: 20s
+            interval: 10s
+            timeout: 10s
+            retries: 20 
+        deploy:
+            resources:
+                limits:
+                    cpus: 1
+                    memory: 8G 
+    globus-ubuntu22:
+        build:
+            context: .
+            dockerfile: globus.ubuntu22.Dockerfile
+        depends_on:
+            irods:
+                condition: service_healthy
+        volumes:
+            - ../../:/irods_client_globus_connector 
+    globus-ubuntu20:
+        build:
+            context: .
+            dockerfile: globus.ubuntu20.Dockerfile
+        depends_on:
+            irods:
+                condition: service_healthy
+        volumes:
+            - ../../:/irods_client_globus_connector 

tests/docker/globus.ubuntu20.Dockerfile

@@ -0,0 +1,61 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true
+
+#### Get and install iRODS repo ####
+RUN apt-get update && apt-get install -y wget gnupg2 lsb-release
+RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add -
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/renci-irods.list
+RUN apt-get update
+
+#### Install icommands - used to set up, validate and tear down tests. ####
+RUN apt-get install -y irods-icommands
+
+#### Install irods-dev, cmake, clang  - used to build the connector ####
+RUN apt-get install -y irods-externals-cmake3.21.4-0
+RUN apt-get install -y irods-externals-clang*
+RUN apt-get install -y irods-dev
+
+#### install basic packages ####
+RUN apt-get install -y curl \
+    vim \
+    ftp \
+    telnet \
+    libcurl4-openssl-dev \
+    unzip \
+    python3 \
+    python3-distro \
+    python3-psutil \
+    python3-jsonschema \
+    python3-requests \
+    python3-pip \
+    python3-pyodbc
+
+#### Get and install globus repo ####
+RUN wget -q https://downloads.globus.org/globus-connect-server/stable/installers/repo/deb/globus-repo_latest_all.deb
+RUN dpkg -i globus-repo_latest_all.deb
+RUN apt-get update
+
+#### Install and configure globus specific things ####
+RUN apt-get install -y globus-gridftp-server-progs \
+    globus-simple-ca \
+    globus-gass-copy-progs \
+    libglobus-common-dev \
+    libglobus-gridftp-server-dev \
+    libglobus-gridmap-callout-error-dev \
+    globus-gsi-cert-utils-progs \
+    globus-proxy-utils
+
+RUN mkdir /iRODS_DSI
+RUN chmod 777 /iRODS_DSI
+
+#### Get and install iRODS repo ####
+RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add -
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/renci-irods.list
+RUN apt-get update
+
+ADD start.globus.run.tests.ubuntu.sh /
+RUN chmod u+x /start.globus.run.tests.ubuntu.sh
+
+ENTRYPOINT "/start.globus.run.tests.ubuntu.sh"

tests/docker/globus.ubuntu22.Dockerfile

@@ -0,0 +1,61 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true
+
+#### Get and install iRODS repo ####
+RUN apt-get update && apt-get install -y wget gnupg2 lsb-release
+RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add -
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/renci-irods.list
+RUN apt-get update
+
+#### Install icommands - used to set up, validate and tear down tests. ####
+RUN apt-get install -y irods-icommands
+
+#### Install irods-dev, cmake, clang  - used to build the connector ####
+RUN apt-get install -y irods-externals-cmake3.21.4-0
+RUN apt-get install -y irods-externals-clang*
+RUN apt-get install -y irods-dev
+
+#### install basic packages ####
+RUN apt-get install -y curl \
+    vim \
+    ftp \
+    telnet \
+    libcurl4-openssl-dev \
+    unzip \
+    python3 \
+    python3-distro \
+    python3-psutil \
+    python3-jsonschema \
+    python3-requests \
+    python3-pip \
+    python3-pyodbc
+
+#### Get and install globus repo ####
+RUN wget -q https://downloads.globus.org/globus-connect-server/stable/installers/repo/deb/globus-repo_latest_all.deb
+RUN dpkg -i globus-repo_latest_all.deb
+RUN apt-get update
+
+#### Install and configure globus specific things ####
+RUN apt-get install -y globus-gridftp-server-progs \
+    globus-simple-ca \
+    globus-gass-copy-progs \
+    libglobus-common-dev \
+    libglobus-gridftp-server-dev \
+    libglobus-gridmap-callout-error-dev \
+    globus-gsi-cert-utils-progs \
+    globus-proxy-utils
+
+RUN mkdir /iRODS_DSI
+RUN chmod 777 /iRODS_DSI
+
+#### Get and install iRODS repo ####
+RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add -
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/renci-irods.list
+RUN apt-get update
+
+ADD start.globus.run.tests.ubuntu.sh /
+RUN chmod u+x /start.globus.run.tests.ubuntu.sh
+
+ENTRYPOINT "/start.globus.run.tests.ubuntu.sh"

tests/docker/irods.ubuntu22.Dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:22.04
+
+ADD start.irods.ubuntu22.sh /
+RUN chmod u+x /start.irods.ubuntu22.sh
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true
+
+#### install basic packages ####
+RUN apt-get update && \
+    apt-get install -y apt-utils apt-transport-https unixodbc unixodbc-dev wget lsb-release sudo \
+                       libssl-dev super lsof postgresql odbc-postgresql libjson-perl gnupg \
+                       vim sudo rsyslog g++ dpkg-dev cdbs libcurl4-openssl-dev \
+                       tig git libpam0g-dev libkrb5-dev libfuse-dev \
+                       libbz2-dev libxml2-dev zlib1g-dev \
+                       make gcc help2man telnet ftp
+
+RUN apt-get install -y python3 \
+    python3-distro \
+    python3-psutil \
+    python3-jsonschema \
+    python3-requests \
+    python3-pip \
+    python3-pyodbc \
+    python3-dev 
+
+#### Get and install iRODS repo ####
+RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add -
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/renci-irods.list
+RUN apt-get update
+
+#### Install iRODS ####
+#ARG irods_version
+#ENV IRODS_VERSION ${irods_version}
+ENV irods_version 4.3.1-0~jammy
+
+RUN apt-get install -y irods-server=${irods_version} irods-dev=${irods_version} irods-database-plugin-postgres=${irods_version} irods-runtime=${irods_version} irods-icommands=${irods_version}
+
+#### Set up ICAT database. ####
+ADD db_commands.txt /
+RUN service postgresql start && su - postgres -c 'psql -f /db_commands.txt'
+
+ENTRYPOINT "/start.irods.ubuntu22.sh"
+

tests/docker/keep_alive.sh

@@ -0,0 +1,2 @@
+#! /bin/bash
+tail -f /dev/null

tests/docker/start.globus.run.tests.ubuntu.sh

@@ -0,0 +1,82 @@
+#! /bin/bash
+
+#### Give root an environment to connect to iRODS as Alice ####
+#### Needed to set up testing.                             ####
+echo 'irods
+1247
+rods
+tempZone
+rods' | iinit
+
+#### Add user1 as a local user for testing ####
+useradd user1 -m -s /usr/bin/bash
+
+#### Give user1 an environment to connect to iRODS ####
+sudo -H -u user1 bash -c "
+echo 'irods
+1247
+user1
+tempZone
+user1' | iinit"
+
+#### configure globus certs ####
+# the folowing seems to be automatic now
+#sudo grid-ca-create -noint  # puts files in /etc/grid-security/certificates
+
+# this seems required to run grid-cert-request
+mkdir /var/adm
+touch /var/adm/wtmp
+touch /var/log/messages
+
+HEX_ID=$(ls /etc/grid-security/certificates/*.0 | cut -d/ -f5 | cut -d. -f1)
+sed -i 's|= policy_match|= policy_anything|g' /etc/grid-security/certificates/globus-host-ssl.conf.${HEX_ID}
+sed -i 's|cond_subjects     globus       .*|cond_subjects     globus       '"'"'"*"'"'"'|g' /etc/grid-security/certificates/${HEX_ID}.signing_policy
+grid-cert-request -ca ${HEX_ID} -nopw -cn `hostname` -force # creates ~/.globus/usercert.pem usercert_request.pem userkey.pem
+cp ~/.globus/userkey.pem /etc/grid-security/hostkey.pem
+cp /etc/grid-security/certificates/${HEX_ID}.0 ~/.globus/${HEX_ID}.0
+cp /etc/grid-security/certificates/${HEX_ID}.signing_policy ~/.globus/${HEX_ID}.signing_policy
+echo globus  | grid-ca-sign -in ~/.globus/usercert_request.pem -out hostcert.pem  # sign the cert
+cp hostcert.pem /etc/grid-security/hostcert.pem
+cp hostcert.pem ~/.globus/usercert.pem
+
+#### Set up grid-mapfile ####
+subject=`openssl x509 -noout -in /etc/grid-security/hostcert.pem -subject | cut -d'=' -f2- | sed -e 's|,|/|g' | sed -e 's|/ |/|g' | sed -e 's/ = /=/g'`
+echo "\"/$subject\" rods" | sudo tee -a /etc/grid-security/grid-mapfile
+
+#### Set up /etc/gridftp.conf also allowing user1 to user anonymous ftp ####
+echo 'port 2811
+$LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/iRODS_DSI"
+$irodsConnectAsAdmin "rods"
+$spOption irods_client_globus_connector
+$numberOfIrodsReadWriteThreads 3
+$irodsParallelFileSizeThresholdBytes 33554432
+$spOption irods_client_globus_connector
+$irodsResourceMap "/etc/resource_mapfile"
+
+load_dsi_module iRODS
+auth_level 4
+
+allow_anonymous 1
+anonymous_names_allowed user1
+anonymous_user user1
+' | tee -a /etc/gridftp.conf
+
+#### Create a resource_mapfile ####
+echo '/tempZone/home/rods/dir1;resc1
+/tempZone/home/rods/dir2;resc2' > /etc/resource_mapfile
+
+### Build and install the gridftp plugin
+mkdir /bld_irods_client_globus_connector
+cd /bld_irods_client_globus_connector
+/opt/irods-externals/cmake3.21.4-0/bin/cmake /irods_client_globus_connector
+make -j package
+dpkg -i *.deb
+
+#### Start gridftp server ####
+/usr/sbin/globus-gridftp-server -allow-root -log-module stdio:buffer=0 -threads 1 -aa -c /etc/gridftp.conf -pidfile /var/run/globus-gridftp-server.pid -log-level trace,info,warn,error -logfile /var/log/gridftp.log -no-detach -config-base-path / &
+
+cd /
+
+#### Run All Tests ####
+cd /irods_client_globus_connector/tests
+python3 run_all_tests.py

tests/docker/start.irods.ubuntu22.sh

@@ -0,0 +1,39 @@
+#! /bin/bash
+
+# Start the Postgres database.
+service postgresql start
+counter=0
+until pg_isready -q
+do
+    sleep 1
+    ((counter += 1))
+done
+echo Postgres took approximately $counter seconds to fully start ...
+
+#### Set up iRODS ####
+/var/lib/irods/scripts/setup_irods.py < /var/lib/irods/packaging/localhost_setup_postgres.input
+
+#### Start iRODS ####
+service irods start
+
+#### Create user1 in iRODS ####
+sudo -H -u irods bash -c "iadmin mkuser user1 rodsuser"
+sudo -H -u irods bash -c "iadmin moduser user1 password user1"
+
+#### Create resc1 and resc2 resources in iRODS ####
+sudo -H -u irods bash -c "iadmin mkresc resc1 unixfilesystem `hostname`:/tmp/resc1"
+sudo -H -u irods bash -c "iadmin mkresc resc2 unixfilesystem `hostname`:/tmp/resc2"
+
+#### Give root an environment to connect to iRODS ####
+echo 'localhost
+1247
+rods
+tempZone
+rods' | iinit
+
+mkdir -p /projects/irods/vsphere-testing/externals/
+
+cd /
+
+#### Keep container running ####
+tail -f /dev/null

tests/ftp_commands.txt

@@ -0,0 +1,4 @@
+open localhost 2811
+user user1 pass
+put abc.txt
+bye