Skip to content

Typing: Add type annotations to distance.py #5784

Typing: Add type annotations to distance.py

Typing: Add type annotations to distance.py #5784

name: End to End integration tests
on:
- pull_request
- push
jobs:
setup:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Update pip
run: python3 -m pip install -U pip setuptools
- name: Install python requirements for matrix_parser.py
run: python3 -m pip install -U PyYAML
- name: Identify branch
id: branch
run: python3 tools/github/workflow/set_branch_output_env.py
- name: Identify Matrix
id: matrix
run: echo "matrix=$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix_integration_tests.yml)" >> $GITHUB_OUTPUT
outputs:
branch: ${{ steps.branch.outputs.branch }}
matrix: ${{ steps.matrix.outputs.matrix }}
integration-tests:
needs: setup
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cfg: ${{ fromJson(needs.setup.outputs.matrix) }}
steps:
- name: Checkout rucio containers repository
uses: actions/checkout@v4
with:
repository: rucio/containers
fetch-depth: 0
- uses: actions/checkout@v4
name: Checkout rucio source
with:
path: dev/rucio
fetch-depth: 0
- name: Select tag for rucio containers
shell: bash
run: |
# Change to cloned rucio/rucio repo
cd $GITHUB_WORKSPACE/dev/rucio
# Get current branch and corresponding latest tag in rucio/rucio repo
BRANCH=$(git rev-parse --abbrev-ref HEAD)
OWNER="${{ github.repository_owner }}"
if [ $OWNER != 'rucio' ]; then
echo "The workflow is running in user ${OWNER}'s fork. Fetching branches and tags from rucio/rucio instead."
git remote add rucio https://github.com/rucio/rucio
git fetch rucio --tags -f
fi
echo "On branch ${BRANCH}"
if [ $BRANCH == 'master' ]; then
GIT_REF="master"
else
GIT_REF=$(git describe --tags --abbrev=0)
IFS=. read major minor micro build <<<"${GIT_REF}"
RELEASE_FAMILY=$major
LATEST_RELEASE_IN_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags/$RELEASE_FAMILY.* | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}')
LATEST_RUCIO_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}' | awk -F'.' '{print $1}')
echo "Release line for ${BRANCH} is ${RELEASE_FAMILY}"
echo "The latest release line for rucio is ${LATEST_RUCIO_RELEASE_FAMILY}"
echo "The latest release in ${RELEASE_FAMILY} is ${LATEST_RELEASE_IN_RELEASE_FAMILY}"
if [ $LATEST_RUCIO_RELEASE_FAMILY = $RELEASE_FAMILY ]; then
GIT_REF='master' # always use containers/master when working on latest rucio/rucio release line
else
GIT_REF=$LATEST_RELEASE_IN_RELEASE_FAMILY # for non-master release line, use the latest rucio/containers tag for the given release family
fi
fi
cd $GITHUB_WORKSPACE
# Check if rucio/containers has a corresponding $GIT_REF
if [ $(git tag -l "$GIT_REF") ]; then
git checkout tags/$GIT_REF
else
echo "Tag $GIT_REF not found in rucio/containers. Integration test containers will be built against the master branch instead."
git checkout master
fi
- name: Use rucio/containers Dockerfile for integration tests
shell: bash
run: |
sed -i 's;RUN git clone .*;COPY ./rucio /tmp/rucio;' $GITHUB_WORKSPACE/dev/alma9.Dockerfile
- name: Build rucio-dev images
id: images
shell: bash
run: |
docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam pull
i=0; until [ "$i" -ge 3 ]; do
IMAGES=$(echo '${{ toJson(matrix.cfg) }}' | $GITHUB_WORKSPACE/dev/rucio/tools/test/build_images.py --output list \
--cache-repo ghcr.io/${{ github.repository }} --branch "${{ needs.setup.outputs.branch }}" \
$GITHUB_WORKSPACE/dev || echo "")
if [[ -n $IMAGES ]]; then break;
else
i=$((i+1)); sleep 5;
echo "::warning::Building images failed, retrying…"
fi
done
docker logout https://ghcr.io
if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi
echo "images=$IMAGES" >> $GITHUB_OUTPUT
- name: Prepare Docker Compose
shell: bash
run: |
docker image ls
sed -i 's;image: docker.io/rucio/rucio-dev.*;image: ${{ fromJSON(steps.images.outputs.images)[0] }};' \
$GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
- name: Start containers
run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam up -d
- name: Initialize tests
shell: bash
run: |
docker exec -t dev_rucio_1 cp etc/rse-accounts.cfg.template etc/rse-accounts.cfg
docker exec -t dev_rucio_1 tools/run_tests.sh -ir
- name: File Upload/Download Test
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rucio_server.py
- name: UploadClient Test
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_upload.py
- name: File Upload/Download Test using 'impl' parameter
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_impl_upload_download.py
- name: Test gfal2 implementation on xrootd protocol
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_gfal2_impl.py
- name: Test Protocol XrootD
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_xrootd.py
- name: Test Protocol SSH (scp)
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_ssh.py
- name: Test Protocol Rsync
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rsync.py
- name: Test Protocol Rclone
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rclone.py
- name: Test Conveyor
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_conveyor.py
- name: Execute transfer and export FTS transfer details
id: tpc
shell: bash
run: |
docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short --export-artifacts-from="test_tpc" tests/test_tpc.py
FTS_LOG_FILE=$(docker exec -t dev_rucio_1 cat /tmp/test_tpc.artifact)
echo "fts3log=$FTS_LOG_FILE" >> $GITHUB_OUTPUT
- name: Verify TPC transfers from FTS logs
shell: bash
run: |
FTS_LOG_FILE=${{ steps.tpc.outputs.fts3log }}
if docker exec -t dev_fts_1 /bin/bash -c "grep -Fq \"3rd pull\" $FTS_LOG_FILE"
then
echo "TPC 3rd party pull verified"
else
echo "TPC failed"
exit 1
fi
- name: Test Token Deletion
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_reaper.py::test_deletion_with_tokens
- name: Print server error logs
if: ${{ success() || failure() }}
shell: bash
run: |
docker exec -t dev_rucio_1 cat /var/log/rucio/httpd_error_log
- name: Archive Upload/Download Test
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_download.py::test_download_from_archive_on_xrd
- name: Test external metadata plugin (mongo)
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaMongo
- name: Test external metadata plugin (postgres)
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaExternalPostgresJSON
- name: Stop containers
run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam down