Fix abacus tests

etc/docker/test/matrix_policy_package_tests.yml

@@ -143,6 +143,9 @@ belleii:
       - rucio_tests/test_impl_upload_download.py
       - rucio_tests/test_rse_protocol_rclone.py
       - rucio_tests/test_rse_protocol_rsync.py
+      - rucio_tests/test_abacus_collection_replica.py
+      - rucio_tests/test_abacus_rse.py
+      - rucio_tests/test_abacus_account.py
     deny:
       - rucio_tests/test_rse_protocol_ssh.py
       - rucio_tests/test_redirect.py
@@ -152,16 +155,13 @@ belleii:
       - rucio_tests/test_bb8.py
       - rucio_tests/test_rse_lfn2path.py
       - rucio_tests/test_s3.py
-      - rucio_tests/test_abacus_collection_replica.py
       - rucio_tests/test_rule.py
       - rucio_tests/test_subscription.py
       - rucio_tests/test_multi_vo.py
       - rucio_tests/test_rse.py
       - rucio_tests/test_naming_convention.py
-      - rucio_tests/test_abacus_rse.py
       - rucio_tests/test_dataset_replicas.py
       - rucio_tests/test_did_meta_plugins.py
-      - rucio_tests/test_did.py
       - rucio_tests/test_replica.py
       - rucio_tests/test_bin_rucio.py
       - rucio_tests/test_dirac.py
@@ -170,7 +170,7 @@ belleii:
       - rucio_tests/test_permission.py
       - rucio_tests/test_curl.py
       - rucio_tests/test_auditor_hdfs.py
-      - rucio_tests/test_abacus_account.py
+      - rucio_tests/test_did.py
   rdbms:
     - postgres14
   python:

lib/rucio/common/utils.py

@@ -693,6 +693,8 @@ def extract_scope_dirac(did, scopes):
 
 def extract_scope_belleii(did, scopes):
     split_did = did.split('/')
+    if did.startswith('/belle/mock/'):
+        return 'mock', did
     if did.startswith('/belle/MC/'):
         if did.startswith('/belle/MC/BG') or \
            did.startswith('/belle/MC/build') or \

lib/rucio/core/permission/belleii.py

@@ -163,7 +163,13 @@ def perm_add_rule(issuer, kwargs, session=None):
     :returns: True if account is allowed, otherwise False
     """
     restricted_scopes = config_get('permission', 'restricted_scopes', raise_exception=False, default=[])
-    if kwargs['account'] == issuer and not kwargs['locked'] and not (restricted_scopes and kwargs['scope'] in restricted_scopes):
+    if kwargs['account'] == issuer and not kwargs.get('locked', False):
+        if kwargs.get('scope') and restricted_scopes and kwargs['scope'] in restricted_scopes:
+            return False
+        if kwargs.get('dids'):
+            for did in kwargs['dids']:
+                if restricted_scopes and did['scope'] in restricted_scopes:
+                    return False
         return True
     return perm_default(issuer, kwargs, session=session) or has_account_attribute(account=issuer, key='rule_admin', session=session)
 
@@ -378,7 +384,13 @@ def perm_add_did(issuer, kwargs, session=None):
     """
     # Check the accounts of the issued rules
     for rule in kwargs.get('rules', []):
-        if not perm_add_rule(issuer, kwargs=rule, session=session):
+        kwargs_rule = rule
+        if 'scope' not in kwargs_rule:
+            if kwargs['scope'] and not isinstance(kwargs['scope'], str):
+                kwargs_rule['scope'] = kwargs['scope'].external
+            else:
+                kwargs_rule['scope'] = kwargs['scope']
+        if not perm_add_rule(issuer, kwargs=kwargs_rule, session=session):
             return False
 
     return perm_default(issuer, kwargs, session=session)\

lib/rucio/tests/common.py

@@ -63,13 +63,15 @@ def scope_name_generator():
     return 'mock_' + str(uuid()).lower()[:16]
 
 
-def did_name_generator(did_type='file'):
+def did_name_generator(did_type='file', name_prefix='', name_suffix='', cnt=0):
     """ Generate random did name.
 
     :returns: A random did name
     """
     if os.getenv('POLICY') == 'belleii':
-        path = '/belle'
+        path = '/belle/mock' + name_prefix
+        if not path[-1] != '/':
+            path += '/'
         path += '/cont_%s' % str(uuid())
         if did_type == 'container':
             return path
@@ -78,9 +80,10 @@ def did_name_generator(did_type='file'):
             return path
         path += '/file_%s' % str(uuid())
         return path
-    if did_type in ['file', 'dataset', 'container']:
-        return '%s_%s' % (did_type, str(uuid()))
-    return 'mock_' + str(uuid())
+    if name_prefix:
+        name = '%s_%s%s' % (name_prefix, cnt, name_suffix)
+        return name
+    return '%s_%s' % (did_type, str(uuid()))
 
 
 def rse_name_generator(size=10):

lib/rucio/tests/temp_factories.py

@@ -31,7 +31,7 @@
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import DIDType
 from rucio.db.sqla.session import transactional_session
-from rucio.tests.common import file_generator, rse_name_generator
+from rucio.tests.common import file_generator, rse_name_generator, did_name_generator
 from sqlalchemy import and_, or_
 
 
@@ -273,8 +273,9 @@ def _random_did(self, scope, name_prefix, name_suffix=''):
         scope = self._sanitize_or_set_scope(scope)
         if not name_prefix:
             name_prefix = 'lfn'
-        name = '%s_%s_%s%s' % (name_prefix, self.base_uuid, len(self.created_dids), name_suffix)
+        name = did_name_generator(did_type=name_prefix, name_prefix='%s_%s' % (name_prefix, self.base_uuid), name_suffix=name_suffix, cnt=len(self.created_dids))
         did = {'scope': scope, 'name': name}
+        print(did)
         self.created_dids.append(did)
         return did
 
@@ -284,6 +285,7 @@ def random_did(self, scope=None, name_prefix=None, name_suffix=''):
 
     def make_dataset(self, scope=None):
         did = self._random_did(scope=scope, name_prefix='dataset')
+        print(did)
         self.client.add_did(scope=did['scope'].external, name=did['name'], did_type=DIDType.DATASET)
         return did
 
@@ -317,8 +319,10 @@ def upload_test_dataset(self, rse_name, scope=None, size=2, nb_files=2):
         self.created_dids.append(did)
         items = list()
         for _ in range(0, nb_files):
+            # TODO : Call did_name_generator
             path = file_generator(size=size)
-            name = os.path.basename(path)
+            # name = os.path.basename(path)
+            name = did_name_generator('file')
             items.append({
                          'path': path,
                          'rse': rse_name,
@@ -329,7 +333,9 @@ def upload_test_dataset(self, rse_name, scope=None, size=2, nb_files=2):
                          'guid': generate_uuid(),
                          })
             did = {'scope': scope, 'name': name}
+            print(did)
             self.created_dids.append(did)
+        print(items)
         self.upload_client.upload(items)
         return items
 

lib/rucio/tests/test_abacus_account.py

@@ -13,89 +13,64 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import unittest
 
 import pytest
 
-from rucio.client.accountclient import AccountClient
-from rucio.client.uploadclient import UploadClient
-from rucio.common.config import config_get_bool
-from rucio.common.types import InternalAccount, InternalScope
-from rucio.common.utils import generate_uuid
+from rucio.common.schema import get_schema_value
 from rucio.core.account import get_usage_history
 from rucio.core.account_counter import update_account_counter_history
 from rucio.core.account_limit import get_local_account_usage, set_local_account_limit
-from rucio.core.rse import get_rse_id
-from rucio.daemons.abacus import account
+from rucio.daemons.abacus.account import account_update
 from rucio.daemons.judge import cleaner
 from rucio.daemons.reaper import reaper
-from rucio.daemons.undertaker import undertaker
 from rucio.db.sqla import models
 from rucio.db.sqla.session import get_session
-from rucio.tests.common import file_generator
-from rucio.tests.common_server import get_vo
 
 
 @pytest.mark.noparallel(reason='uses daemon, failing in parallel to other tests, updates account')
-class TestAbacusAccount(unittest.TestCase):
-    rse = 'MOCK4'
-    file_sizes = 2
-    vo = {}
+class TestAbacusAccount2():
 
-    @classmethod
-    def setUpClass(cls):
-        cls.upload_client = UploadClient()
-        cls.account_client = AccountClient()
-        cls.session = get_session()
-
-        if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
-            cls.vo = {'vo': get_vo()}
-
-        cls.account = InternalAccount('root', **cls.vo)
-        cls.scope = InternalScope('mock', **cls.vo)
-        cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
-
-    @classmethod
-    def tearDownClass(cls):
-        undertaker.run(once=True)
-        cleaner.run(once=True)
-        if cls.vo:
-            reaper.run(once=True, include_rses='vo=%s&(%s)' % (cls.vo['vo'], cls.rse), greedy=True)
-        else:
-            reaper.run(once=True, include_rses=cls.rse, greedy=True)
-
-    def test_abacus_account(self):
+    def test_abacus_account(self, vo, root_account, mock_scope, rse_factory, did_factory, rucio_client):
         """ ABACUS (ACCOUNT): Test update of account usage """
-        self.session.query(models.UpdatedAccountCounter).delete()  # pylint: disable=no-member
-        self.session.query(models.AccountUsage).delete()  # pylint: disable=no-member
-        self.session.commit()  # pylint: disable=no-member
+        session = get_session()
+        session.query(models.UpdatedAccountCounter).delete()  # pylint: disable=no-member
+        session.query(models.AccountUsage).delete()  # pylint: disable=no-member
+        session.commit()  # pylint: disable=no-member
 
         # Upload files -> account usage should increase
-        self.files = [{'did_scope': self.scope.external, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1} for i in range(0, 2)]
-        self.upload_client.upload(self.files)
-        [os.remove(file['path']) for file in self.files]
-        account.run(once=True)
-        account_usage = get_local_account_usage(account=self.account, rse_id=self.rse_id)[0]
-        assert account_usage['bytes'] == len(self.files) * self.file_sizes
-        assert account_usage['files'] == len(self.files)
+        file_sizes = 2
+        nfiles = 2
+        rse, rse_id = rse_factory.make_posix_rse()
+        dids = did_factory.upload_test_dataset(rse_name=rse, scope=mock_scope.external, size=file_sizes, nb_files=nfiles)
+        dataset = dids[0]['dataset_name']
+        activity = get_schema_value('ACTIVITY')['enum'][0]
+        rucio_client.add_replication_rule([{'scope': mock_scope.external, 'name': dataset}], 1, rse, lifetime=-1, activity=activity)
+        account_update(once=True)
+        account_usage = get_local_account_usage(account=root_account, rse_id=rse_id)[0]
+        assert account_usage['bytes'] == nfiles * file_sizes
+        assert account_usage['files'] == nfiles
 
         # Update and check the account history with the core method
-        update_account_counter_history(account=self.account, rse_id=self.rse_id)
-        usage_history = get_usage_history(rse_id=self.rse_id, account=self.account)
-        assert usage_history[-1]['bytes'] == len(self.files) * self.file_sizes
-        assert usage_history[-1]['files'] == len(self.files)
+        update_account_counter_history(account=root_account, rse_id=rse_id)
+        usage_history = get_usage_history(rse_id=rse_id, account=root_account)
+        assert usage_history[-1]['bytes'] == nfiles * file_sizes
+        assert usage_history[-1]['files'] == nfiles
 
         # Check the account history with the client
-        usage_history = self.account_client.get_account_usage_history(rse=self.rse, account=self.account.external)
-        assert usage_history[-1]['bytes'] == len(self.files) * self.file_sizes
-        assert usage_history[-1]['files'] == len(self.files)
+        usage_history = rucio_client.get_account_usage_history(rse=rse, account=root_account.external)
+        assert usage_history[-1]['bytes'] == nfiles * file_sizes
+        assert usage_history[-1]['files'] == nfiles
 
         # Delete rules -> account usage should decrease
         cleaner.run(once=True)
-        account.run(once=True)
+        account_update(once=True)
         # set account limit because return value of get_local_account_usage differs if a limit is set or not
-        set_local_account_limit(account=self.account, rse_id=self.rse_id, bytes_=10)
-        account_usages = get_local_account_usage(account=self.account, rse_id=self.rse_id)[0]
+        set_local_account_limit(account=root_account, rse_id=rse_id, bytes_=10)
+        account_usages = get_local_account_usage(account=root_account, rse_id=rse_id)[0]
         assert account_usages['bytes'] == 0
         assert account_usages['files'] == 0
+
+        if vo:
+            reaper.run(once=True, include_rses='vo=%s&(%s)' % (str(vo), rse), greedy=True)
+        else:
+            reaper.run(once=True, include_rses=rse, greedy=True)