Create base Daemon class and refactor Undertaker, Reaper, Abacus Account and Judge Repairer daemons to inherit from base class #5774

Workflow file for this run

.github/workflows/integration_tests.yml at 91599a5

	name: End to End integration tests

	on:
	- pull_request
	- push

	jobs:
	setup:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Update pip
	run: python3 -m pip install -U pip setuptools
	- name: Install python requirements for matrix_parser.py
	run: python3 -m pip install -U PyYAML
	- name: Identify branch
	id: branch
	run: python3 tools/github/workflow/set_branch_output_env.py
	- name: Identify Matrix
	id: matrix
	run: echo "matrix=$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix_integration_tests.yml)" >> $GITHUB_OUTPUT
	outputs:
	branch: ${{ steps.branch.outputs.branch }}
	matrix: ${{ steps.matrix.outputs.matrix }}

	integration-tests:
	needs: setup
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	cfg: ${{ fromJson(needs.setup.outputs.matrix) }}
	steps:
	- name: Checkout rucio containers repository
	uses: actions/checkout@v4
	with:
	repository: rucio/containers
	fetch-depth: 0
	- uses: actions/checkout@v4
	name: Checkout rucio source
	with:
	path: dev/rucio
	fetch-depth: 0
	- name: Select tag for rucio containers
	shell: bash
	run: \|
	# Change to cloned rucio/rucio repo
	cd $GITHUB_WORKSPACE/dev/rucio

	# Get current branch and corresponding latest tag in rucio/rucio repo
	BRANCH=$(git rev-parse --abbrev-ref HEAD)
	OWNER="${{ github.repository_owner }}"

	if [ $OWNER != 'rucio' ]; then
	echo "The workflow is running in user ${OWNER}'s fork. Fetching branches and tags from rucio/rucio instead."
	git remote add rucio https://github.com/rucio/rucio
	git fetch rucio --tags -f
	fi

	echo "On branch ${BRANCH}"
	if [ $BRANCH == 'master' ]; then
	GIT_REF="master"
	else
	GIT_REF=$(git describe --tags --abbrev=0)
	IFS=. read major minor micro build <<<"${GIT_REF}"

	RELEASE_FAMILY=$major
	LATEST_RELEASE_IN_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags/$RELEASE_FAMILY.* \| sort -k 1.11V \| tail -1 \| awk -F'/' '{print $3}')
	LATEST_RUCIO_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags \| sort -k 1.11V \| tail -1 \| awk -F'/' '{print $3}' \| awk -F'.' '{print $1}')

	echo "Release line for ${BRANCH} is ${RELEASE_FAMILY}"
	echo "The latest release line for rucio is ${LATEST_RUCIO_RELEASE_FAMILY}"
	echo "The latest release in ${RELEASE_FAMILY} is ${LATEST_RELEASE_IN_RELEASE_FAMILY}"

	if [ $LATEST_RUCIO_RELEASE_FAMILY = $RELEASE_FAMILY ]; then
	GIT_REF='master' # always use containers/master when working on latest rucio/rucio release line
	else
	GIT_REF=$LATEST_RELEASE_IN_RELEASE_FAMILY # for non-master release line, use the latest rucio/containers tag for the given release family
	fi

	fi

	cd $GITHUB_WORKSPACE

	# Check if rucio/containers has a corresponding $GIT_REF
	if [ $(git tag -l "$GIT_REF") ]; then
	git checkout tags/$GIT_REF
	else
	echo "Tag $GIT_REF not found in rucio/containers. Integration test containers will be built against the master branch instead."
	git checkout master
	fi
	- name: Use rucio/containers Dockerfile for integration tests
	shell: bash
	run: \|
	sed -i 's;RUN git clone .*;COPY ./rucio /tmp/rucio;' $GITHUB_WORKSPACE/dev/alma9.Dockerfile
	- name: Build rucio-dev images
	id: images
	shell: bash
	run: \|
	docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
	docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam pull
	i=0; until [ "$i" -ge 3 ]; do
	IMAGES=$(echo '${{ toJson(matrix.cfg) }}' \| $GITHUB_WORKSPACE/dev/rucio/tools/test/build_images.py --output list \
	--cache-repo ghcr.io/${{ github.repository }} --branch "${{ needs.setup.outputs.branch }}" \
	$GITHUB_WORKSPACE/dev \|\| echo "")
	if [[ -n $IMAGES ]]; then break;
	else
	i=$((i+1)); sleep 5;
	echo "::warning::Building images failed, retrying…"
	fi
	done
	docker logout https://ghcr.io
	if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi
	echo "images=$IMAGES" >> $GITHUB_OUTPUT
	- name: Prepare Docker Compose
	shell: bash
	run: \|
	docker image ls
	sed -i 's;image: docker.io/rucio/rucio-dev.*;image: ${{ fromJSON(steps.images.outputs.images)[0] }};' \
	$GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
	- name: Start containers
	run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam up -d
	- name: Initialize tests
	shell: bash
	run: \|
	docker exec -t dev_rucio_1 cp etc/rse-accounts.cfg.template etc/rse-accounts.cfg
	docker exec -t dev_rucio_1 tools/run_tests.sh -ir
	- name: File Upload/Download Test
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rucio_server.py
	- name: UploadClient Test
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_upload.py
	- name: File Upload/Download Test using 'impl' parameter
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_impl_upload_download.py
	- name: Test gfal2 implementation on xrootd protocol
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_gfal2_impl.py
	- name: Test Protocol XrootD
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_xrootd.py
	- name: Test Protocol SSH (scp)
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_ssh.py
	- name: Test Protocol Rsync
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rsync.py
	- name: Test Protocol Rclone
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rclone.py
	- name: Test Conveyor
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_conveyor.py
	- name: Execute transfer and export FTS transfer details
	id: tpc
	shell: bash
	run: \|
	docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short --export-artifacts-from="test_tpc" tests/test_tpc.py
	FTS_LOG_FILE=$(docker exec -t dev_rucio_1 cat /tmp/test_tpc.artifact)
	echo "fts3log=$FTS_LOG_FILE" >> $GITHUB_OUTPUT
	- name: Verify TPC transfers from FTS logs
	shell: bash
	run: \|
	FTS_LOG_FILE=${{ steps.tpc.outputs.fts3log }}
	if docker exec -t dev_fts_1 /bin/bash -c "grep -Fq \"3rd pull\" $FTS_LOG_FILE"
	then
	echo "TPC 3rd party pull verified"
	else
	echo "TPC failed"
	exit 1
	fi
	- name: Test Token Deletion
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_reaper.py::test_deletion_with_tokens
	- name: Print server error logs
	if: ${{ success() \|\| failure() }}
	shell: bash
	run: \|
	docker exec -t dev_rucio_1 cat /var/log/rucio/httpd_error_log
	- name: Archive Upload/Download Test
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_download.py::test_download_from_archive_on_xrd
	- name: Test external metadata plugin (mongo)
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaMongo
	- name: Test external metadata plugin (postgres)
	run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaExternalPostgresJSON
	- name: Stop containers
	run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam down

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Create base Daemon class and refactor Undertaker, Reaper, Abacus Account and Judge Repairer daemons to inherit from base class #5774

Workflow file

Create base Daemon class and refactor Undertaker, Reaper, Abacus Account and Judge Repairer daemons to inherit from base class #5774

Jobs

Run details

Workflow file for this run