rucio · jamesp-epcc · Mar 14, 2024
diff --git a/.github/workflows/vo_tests.yml b/.github/workflows/vo_tests.yml
@@ -20,9 +20,13 @@ jobs:
       - name: Identify Matrix
         id: matrix
         run: echo "matrix=$(./tools/test/votest_helper.py)" >> $GITHUB_OUTPUT
+      - name: Identify DUNE Matrix
+        id: dune-matrix
+        run: echo "::set-output name=matrix::$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix_dune_tests.yml)"
     outputs:
       branch: ${{ steps.branch.outputs.branch }}
       matrix: ${{ steps.matrix.outputs.matrix }}
+      dune-matrix: ${{ steps.dune-matrix.outputs.matrix }}
   test:
     needs: setup
     runs-on: ubuntu-latest
@@ -38,3 +42,128 @@ jobs:
           cfg: ${{ toJson(matrix.cfg) }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           branch: ${{ needs.setup.outputs.branch }}
+  dune-tests:
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cfg: ${{ fromJson(needs.setup.outputs.dune-matrix) }}
+    steps:
+      - name: Checkout rucio containers repository
+        uses: actions/checkout@v3
+        with:
+          repository: rucio/containers
+          fetch-depth: 0
+      - uses: actions/checkout@v3
+        name: Checkout rucio source
+        with:
+          path: dev/rucio
+          fetch-depth: 0
+      - uses: actions/checkout@v3
+        name: Checkout DUNE policy package
+        with:
+          repository: jamesp-epcc/DUNERucioPolicy
+          path: dev/DUNERucioPolicy
+          fetch-depth: 0
+      - name: Select tag for rucio containers
+        shell: bash
+        run: |
+          # Change to cloned rucio/rucio repo
+          cd $GITHUB_WORKSPACE/dev/rucio
+
+          # Get current branch and corresponding latest tag in rucio/rucio repo
+          BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          OWNER="${{ github.repository_owner }}"
+
+          if [ $OWNER != 'rucio' ]; then
+            echo "The workflow is running in user ${OWNER}'s fork. Fetching branches and tags from rucio/rucio instead."
+            git remote add rucio https://github.com/rucio/rucio
+            git fetch rucio --tags -f
+          fi
+
+          echo "On branch ${BRANCH}"
+          if [ $BRANCH == 'master' ]; then
+              GIT_REF="master"
+          else
+              GIT_REF=$(git describe --tags --abbrev=0)
+              IFS=. read major minor micro build <<<"${GIT_REF}"
+
+              RELEASE_FAMILY=$major
+              LATEST_RELEASE_IN_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags/$RELEASE_FAMILY.* | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}')
+              LATEST_RUCIO_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}' | awk -F'.' '{print $1}')
+
+              echo "Release line for ${BRANCH} is ${RELEASE_FAMILY}"
+              echo "The latest release line for rucio is ${LATEST_RUCIO_RELEASE_FAMILY}"
+              echo "The latest release in ${RELEASE_FAMILY} is ${LATEST_RELEASE_IN_RELEASE_FAMILY}"
+
+              if [ $LATEST_RUCIO_RELEASE_FAMILY = $RELEASE_FAMILY ]; then
+                  GIT_REF='master' # always use containers/master when working on latest rucio/rucio release line
+              else
+                  GIT_REF=$LATEST_RELEASE_IN_RELEASE_FAMILY # for non-master release line, use the latest rucio/containers tag for the given release family
+              fi
+
+          fi
+
+          cd $GITHUB_WORKSPACE
+
+          # Check if rucio/containers has a corresponding $GIT_REF
+          if [ $(git tag -l "$GIT_REF") ]; then
+              git checkout tags/$GIT_REF
+          else
+              echo "Tag $GIT_REF not found in rucio/containers. DUNE test containers will be built against the master branch instead."
+              git checkout master
+          fi
+      - name: Use rucio/containers Dockerfile for DUNE tests
+        shell: bash
+        run: |
+          sed -i 's;RUN git clone .*;COPY ./rucio /tmp/rucio;' $GITHUB_WORKSPACE/dev/alma9.Dockerfile
+
+          # Include DUNE policy package and dependency
+          sed -i '/COPY rse_repository.json/ a\COPY DUNERucioPolicy /usr/local/lib/python3.9/site-packages/DUNERucioPolicy\nRUN python3 -m pip install --no-cache --upgrade metacat\nENV METACAT_SERVER_URL http://dev_metacat_1:8080/\nENV DUNE_TEST True' $GITHUB_WORKSPACE/dev/alma9.Dockerfile
+      - name: Build rucio-dev images
+        id: images
+        shell: bash
+        run: |
+          docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+          docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam pull
+          i=0; until [ "$i" -ge 3 ]; do
+            IMAGES=$(echo '${{ toJson(matrix.cfg) }}' | $GITHUB_WORKSPACE/dev/rucio/tools/test/build_images.py --output list \
+                --cache-repo ghcr.io/${{ github.repository }} --branch "${{ needs.setup.outputs.branch }}" \
+                $GITHUB_WORKSPACE/dev || echo "")
+            if [[ -n $IMAGES ]]; then break;
+            else
+              i=$((i+1)); sleep 5;
+              echo "::warning::Building images failed, retrying…"
+            fi
+          done
+          docker logout https://ghcr.io
+          if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi
+          echo "::set-output name=images::$IMAGES"
+      - name: Prepare Docker Compose
+        shell: bash
+        run: |
+          docker image ls
+          sed -i 's;image: docker.io/rucio/rucio-dev.*;image: ${{ fromJSON(steps.images.outputs.images)[0] }};' \
+              $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
+          echo '  metacat:' >> $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
+          echo '    image: docker.io/jamespepcc/metacat_test' >> $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
+          echo '    ports:' >> $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
+          echo '      - "127.0.0.1:8000:8080"' >> $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
+      - name: Start containers
+        run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam up -d
+      - name: Initialize tests
+        shell: bash
+        run: |
+          # Load policy package from config file
+          docker exec -t dev_rucio_1 sed -i '/schema = atlas/ a\package = DUNERucioPolicy' /opt/rucio/etc/rucio.cfg
+
+          # Use DUNE's lfn2pfn algorithm
+          docker exec -t dev_rucio_1 sed -i 's/lfn2pfn_algorithm_default = hash/lfn2pfn_algorithm_default = DUNE/' /opt/rucio/etc/rucio.cfg
+          docker exec -t dev_rucio_1 cp etc/rse-accounts.cfg.template etc/rse-accounts.cfg
+          docker exec -t dev_rucio_1 python tests/dune_init_metacat.py
+          docker exec -t dev_rucio_1 tools/run_tests.sh -ir
+      - name: DUNE specific tests
+        run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_dune.py
+      - name: Stop containers
+        run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam down
diff --git a/etc/docker/test/matrix_dune_tests.yml b/etc/docker/test/matrix_dune_tests.yml
@@ -0,0 +1,11 @@
+dists:
+  - id: alma9
+    allow:
+      python:
+        - "3.9"
+python:
+  - "3.9"
+suites:
+  - dune-test
+image_identifier:
+  - dune-test
diff --git a/lib/rucio/tests/common.py b/lib/rucio/tests/common.py
@@ -37,6 +37,8 @@
                                   reason="does not work for multiVO")
 skip_non_belleii = pytest.mark.skipif(not ('POLICY' in os.environ and os.environ['POLICY'] == 'belleii'),
                                       reason="specific belleii tests")
+skip_non_dune = pytest.mark.skipif('DUNE_TEST' not in os.environ,
+                                   reason="specific DUNE tests")
 
 
 def is_influxdb_available() -> bool:

diff --git a/tests/dune_init_metacat.py b/tests/dune_init_metacat.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from metacat.webapi import MetaCatClient
+
+client = MetaCatClient("http://dev_metacat_1:8080/")
+client.login_password("admin", "admin")
+
+# add the containers, datasets and files created by run_tests.sh -ir
+# to MetaCat so that the DUNE add_did permission check passes
+client.create_namespace("test")
+
+client.create_dataset("test:container")
+client.create_dataset("test:dataset1")
+client.create_dataset("test:dataset2")
+client.create_dataset("test:dataset3")
+
+client.add_child_dataset("test:container", "test:dataset1")
+client.add_child_dataset("test:container", "test:dataset2")
+
+client.declare_file(did="test:file1", dataset_did="test:dataset1")
+client.declare_file(did="test:file2", dataset_did="test:dataset1")
+client.declare_file(did="test:file3", dataset_did="test:dataset2")
+client.declare_file(did="test:file4", dataset_did="test:dataset2")
+client.add_files("test:dataset3", file_list=[{"did": "test:file4"}])