Testing: Re-activate integration tests. #2311 (#4578)

* Testing: Activate XRD archive download test on integration. #2311 The dependency commit was merged. * Testing: search rse-repository.json and rse_account.cfg in config dirs otherwise tests fail when not run from the project base dir Integration tests got silently disabled because the script which runs tests was updated to "cd" into a different directory. * Testing: mark protocol xrootd tests noparallel. #2311 Each test creates a directory and then removes it. One test can remove the directory while another test needs it.
rucio · Apr 27, 2021 · 0003068 · 0003068
1 parent 753e658
commit 0003068
Show file tree

Hide file tree

Showing 11 changed files with 59 additions and 61 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -74,6 +74,8 @@ jobs:
           docker exec -t dev_rucio_1 tools/run_tests_docker.sh -ir
       - name: File Upload/Download Test
         run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short test_rucio_server.py
+      - name: Archive Upload/Download Test
+        run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short test_download.py::test_download_from_archive_on_xrd
       - name: Test Protocol XrootD
         run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short test_rse_protocol_xrootd.py
       - name: Stop containers

diff --git a/lib/rucio/tests/common.py b/lib/rucio/tests/common.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2012-2020 CERN
+# Copyright 2012-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,9 +24,11 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 from __future__ import print_function
 
+import json
 import contextlib
 import itertools
 import os
@@ -38,8 +40,10 @@
 from six import PY3
 
 from rucio.common.utils import generate_uuid as uuid, execute
+from rucio.common.config import get_config_dirs
 
-skip_rse_tests_with_accounts = pytest.mark.skipif(not os.path.exists('etc/rse-accounts.cfg'), reason='fails if no rse-accounts.cfg found')
+skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg')) for d in get_config_dirs()),
+                                                  reason='fails if no rse-accounts.cfg found')
 skiplimitedsql = pytest.mark.skipif('RDBMS' in os.environ and (os.environ['RDBMS'] == 'sqlite' or os.environ['RDBMS'] == 'mysql5'),
                                     reason="does not work in SQLite or MySQL 5, because of missing features")
 
@@ -164,3 +168,9 @@ class Mime:
     JSON = 'application/json'
     JSON_STREAM = 'application/x-json-stream'
     BINARY = 'application/octet-stream'
+
+
+def load_test_conf_file(file_name):
+    config_dir = next(filter(lambda d: os.path.exists(os.path.join(d, file_name)), get_config_dirs()))
+    with open(os.path.join(config_dir, file_name)) as f:
+        return json.load(f)
diff --git a/lib/rucio/tests/rsemgr_api_test.py b/lib/rucio/tests/rsemgr_api_test.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,17 +22,18 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Mayank Sharma <mayank.sharma@cern.ch>, 2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 from __future__ import print_function
 
-import json
 import os
 import os.path
 import tempfile
 
 from rucio.common.utils import adler32
 from rucio.rse import rsemanager as mgr
-from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file
 
 try:
     from exceptions import NotImplementedError
@@ -55,8 +56,7 @@ class MgrTestCases:
     def __init__(self, tmpdir, rse_tag, user, static_file, vo='def'):
         self.rse_settings = mgr.get_rse_info(rse=rse_tag, vo=vo)
         try:
-            with open('etc/rse-accounts.cfg') as f:
-                data = json.load(f)
+            data = load_test_conf_file('rse-accounts.cfg')
             self.rse_settings['credentials'] = data[rse_tag]
         except KeyError:
             print('No credentials found for this RSE.')

diff --git a/lib/rucio/tests/test_download.py b/lib/rucio/tests/test_download.py
@@ -37,6 +37,7 @@
 from rucio.core import did as did_core
 from rucio.rse import rsemanager as rsemgr
 from rucio.rse.protocols.posix import Default as PosixProtocol
+from rucio.tests.common import skip_rse_tests_with_accounts
 
 
 @pytest.fixture
@@ -220,9 +221,9 @@ def test_download_multiple(rse_factory, did_factory, download_client):
         )
 
 
-@pytest.mark.xfail(reason='XRD1 must be initialized https://github.com/rucio/rucio/pull/4165/')
 @pytest.mark.dirty
 @pytest.mark.noparallel(reason='uses pre-defined XRD1 RSE, may fails when run in parallel')  # TODO: verify if it really fails
+@skip_rse_tests_with_accounts
 def test_download_from_archive_on_xrd(did_factory, download_client, did_client):
     scope = 'test'
     rse = 'XRD1'

diff --git a/lib/rucio/tests/test_rse_protocol_gfal2.py b/lib/rucio/tests/test_rse_protocol_gfal2.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014-2020 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2014-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,8 +17,8 @@
 # - Wen Guan <wen.guan@cern.ch>, 2014
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2014-2015
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
-import json
 import os
 import shutil
 import tempfile
@@ -30,7 +30,7 @@
 from rucio.common import exception
 from rucio.common.utils import execute
 from rucio.rse import rsemanager as mgr
-from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file
 from rucio.tests.rsemgr_api_test import MgrTestCases
 
 
@@ -52,8 +52,7 @@ def setUpClass(cls):
         for f in MgrTestCases.files_local:
             shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
 
-        with open('etc/rse_repository.json') as f:
-            data = json.load(f)
+        data = load_test_conf_file('rse_repository.json')
         prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
         hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
         if hostname.count("://"):
@@ -90,8 +89,7 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         """GFAL2 (RSE/PROTOCOLS): Removing created directorie s and files"""
-        with open('etc/rse_repository.json') as f:
-            data = json.load(f)
+        data = load_test_conf_file('rse_repository.json')
         prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
         hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
         if hostname.count("://"):

diff --git a/lib/rucio/tests/test_rse_protocol_posix.py b/lib/rucio/tests/test_rse_protocol_posix.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,10 +19,10 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2017
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 from __future__ import print_function
 
-import json
 import os
 import shutil
 import tempfile
@@ -33,7 +33,7 @@
 
 from rucio.common import exception
 from rucio.rse import rsemanager as mgr
-from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file
 from rucio.tests.rsemgr_api_test import MgrTestCases
 
 
@@ -59,8 +59,7 @@ def setUpClass(cls):
         for f in MgrTestCases.files_local:
             shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
 
-        with open('etc/rse_repository.json') as f:
-            data = json.load(f)
+        data = load_test_conf_file('rse_repository.json')
         prefix = data['MOCK-POSIX']['protocols']['supported']['file']['prefix']
         try:
             os.mkdir(prefix)
@@ -80,8 +79,7 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         """POSIX (RSE/PROTOCOLS): Removing created directorie s and files """
-        with open('etc/rse_repository.json') as f:
-            data = json.load(f)
+        data = load_test_conf_file('rse_repository.json')
         prefix = data['MOCK-POSIX']['protocols']['supported']['file']['prefix']
         shutil.rmtree(prefix)
         shutil.rmtree(cls.tmpdir)

diff --git a/lib/rucio/tests/test_rse_protocol_s3boto.py b/lib/rucio/tests/test_rse_protocol_s3boto.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014-2020 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2014-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,12 +19,10 @@
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
-#
-# PY3K COMPATIBLE
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 from __future__ import print_function
 
-import json
 import os
 import tempfile
 import unittest
@@ -37,7 +35,7 @@
 
 from rucio.common import exception
 from rucio.rse import rsemanager as mgr
-from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file
 from rucio.tests.rsemgr_api_test import MgrTestCases
 
 try:
@@ -92,8 +90,7 @@ def setUpClass(cls):
         rse_tag = 'BNL-BOTO'
         rse_settings = mgr.get_rse_info(rse_tag)
         try:
-            with open('etc/rse-accounts.cfg') as f:
-                data = json.load(f)
+            data = load_test_conf_file('rse-accounts.cfg')
             rse_settings['credentials'] = data[rse_tag]
         except KeyError:
             print('No credentials found for this RSE.')
@@ -131,8 +128,7 @@ def tearDownClass(cls):
         rse_tag = 'BNL-BOTO'
         rse_settings = mgr.get_rse_info(rse_tag)
         try:
-            with open('etc/rse-accounts.cfg') as f:
-                data = json.load(f)
+            data = load_test_conf_file('rse-accounts.cfg')
             rse_settings['credentials'] = data[rse_tag]
         except KeyError:
             print('No credentials found for this RSE.')

diff --git a/lib/rucio/tests/test_rse_protocol_sftp.py b/lib/rucio/tests/test_rse_protocol_sftp.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,8 +20,8 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012
 # - Martin Barisits <martin.barisits@cern.ch>, 2017
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
-import json
 import os
 import shutil
 import tempfile
@@ -33,7 +33,7 @@
 
 from rucio.common import exception
 from rucio.rse import rsemanager as mgr
-from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file
 from rucio.tests.rsemgr_api_test import MgrTestCases
 
 
@@ -56,12 +56,10 @@ def setUpClass(cls):
             os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, fil))
 
         # Load local credentials from file
-        with open('etc/rse-accounts.cfg') as fil:
-            data = json.load(fil)
+        data = load_test_conf_file('rse-accounts.cfg')
         credentials = data['LXPLUS']
         lxplus = pysftp.Connection(**credentials)
-        with open('etc/rse_repository.json') as fil:
-            prefix = json.load(fil)['LXPLUS']['protocols']['supported']['sftp']['prefix']
+        prefix = load_test_conf_file('rse_repository.json')['LXPLUS']['protocols']['supported']['sftp']['prefix']
         lxplus.execute('mkdir %s' % prefix)
         lxplus.execute('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % prefix)
         cls.static_file = 'sftp://lxplus.cern.ch:22%sdata.raw' % prefix
@@ -77,12 +75,10 @@ def tearDownClass(cls):
         """SFTP (RSE/PROTOCOLS): Removing created directorie s and files """
         # Load local creditentials from file
         credentials = {}
-        with open('etc/rse-accounts.cfg') as fil:
-            data = json.load(fil)
+        data = load_test_conf_file('rse-accounts.cfg')
         credentials = data['LXPLUS']
         lxplus = pysftp.Connection(**credentials)
-        with open('etc/rse_repository.json') as fil:
-            prefix = json.load(fil)['LXPLUS']['protocols']['supported']['sftp']['prefix']
+        prefix = load_test_conf_file('rse_repository.json')['LXPLUS']['protocols']['supported']['sftp']['prefix']
         lxplus.execute('rm -rf %s' % prefix)
         lxplus.close()
         shutil.rmtree(cls.tmpdir)

diff --git a/lib/rucio/tests/test_rse_protocol_srm.py b/lib/rucio/tests/test_rse_protocol_srm.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014-2020 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2014-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,8 +17,8 @@
 # - Wen Guan <wen.guan@cern.ch>, 2014
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2014
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
-import json
 import os
 import shutil
 import tempfile
@@ -30,7 +30,7 @@
 from rucio.common import exception
 from rucio.common.utils import execute
 from rucio.rse import rsemanager as mgr
-from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file
 from rucio.tests.rsemgr_api_test import MgrTestCases
 
 
@@ -52,8 +52,7 @@ def setUpClass(cls):
         for f in MgrTestCases.files_local:
             shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
 
-        with open('etc/rse_repository.json') as f:
-            data = json.load(f)
+        data = load_test_conf_file('rse_repository.json')
         prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
         hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
         if hostname.count("://"):
@@ -83,8 +82,7 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         """SRM (RSE/PROTOCOLS): Removing created directorie s and files"""
-        with open('etc/rse_repository.json') as f:
-            data = json.load(f)
+        data = load_test_conf_file('rse_repository.json')
         prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
         hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
         if hostname.count("://"):