Core & Internals: Add setting did meta in bulk; rucio#3847

Additionally use set_metadata_bulk in add_did and add_replica.

lib/rucio/api/did.py

@@ -1,25 +1,35 @@
-'''
-  Copyright European Organization for Nuclear Research (CERN)
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  You may not use this file except in compliance with the License.
-  You may obtain a copy of the
-  License at http://www.apache.org/licenses/LICENSE-2.0
-
-  Authors:
-  - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2017
-  - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2015
-  - Yun-Pin Sun, <yun-pin.sun@cern.ch>, 2013
-  - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2020
-  - Martin Barisits, <martin.barisits@cern.ch>, 2014-2015
-  - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018-2019
-  - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
-  - Ruturaj Gujar, <ruturaj.gujar23@gmail.com>, 2019
-  - Eli Chadwick, <eli.chadwick@stfc.ac.uk>, 2020
-  - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
-
-  PY3K COMPATIBLE
-'''
+# -*- coding: utf-8 -*-
+# Copyright 2013-2020 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
+# - Ralph Vigne <ralph.vigne@cern.ch>, 2013
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2015
+# - Yun-Pin Sun <winter0128@gmail.com>, 2013
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2013
+# - Martin Barisits <martin.barisits@cern.ch>, 2014-2020
+# - asket <asket.agarwal96@gmail.com>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Eric Vaandering <ewv@fnal.gov>, 2020
+# - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2020
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 
 from __future__ import print_function
 
@@ -421,6 +431,30 @@ def set_metadata(scope, name, key, value, issuer, recursive=False, vo='def'):
     return did.set_metadata(scope=scope, name=name, key=key, value=value, recursive=recursive)
 
 
+def set_metadata_bulk(scope, name, meta, issuer, recursive=False, vo='def'):
+    """
+    Add metadata to data did.
+
+    :param scope: The scope name.
+    :param name: The data identifier name.
+    :param meta: the key-values.
+    :param issuer: The issuer account.
+    :param recursive: Option to propagate the metadata update to content.
+    :param vo: The VO to act on.
+    """
+    kwargs = {'scope': scope, 'name': name, 'meta': meta, 'issuer': issuer}
+
+    for key in meta:
+        if key in RESERVED_KEYS:
+            raise rucio.common.exception.AccessDenied('Account %s can not change the value of the metadata key %s to data identifier %s:%s' % (issuer, key, scope, name))
+
+    if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='set_metadata_bulk', kwargs=kwargs):
+        raise rucio.common.exception.AccessDenied('Account %s can not add metadata to data identifier %s:%s' % (issuer, scope, name))
+
+    scope = InternalScope(scope, vo=vo)
+    return did.set_metadata_bulk(scope=scope, name=name, meta=meta, recursive=recursive)
+
+
 def get_metadata(scope, name, plugin='DID_COLUMN', vo='def'):
     """
     Get data identifier metadata

lib/rucio/client/didclient.py

@@ -1,4 +1,5 @@
-# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2013-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,21 +14,23 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2015
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2013-2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2020
-# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
-# - Eric Vaandering <ericvaandering@gmail.com>, 2018
-# - Asket Agarwal <asket.agarwal96@gmail.com>, 2018
+# - Eric Vaandering <ewv@fnal.gov>, 2018-2020
+# - asket <asket.agarwal96@gmail.com>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
-#
-# PY3K COMPATIBLE
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2020
+# - Alan Malta Rodrigues <alan.malta@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 
 from __future__ import print_function
 
@@ -477,6 +480,26 @@ def set_metadata(self, scope, name, key, value, recursive=False):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
+    def set_metadata_bulk(self, scope, name, meta, recursive=False):
+        """
+        Set data identifier metadata in bulk.
+
+        :param scope: The scope name.
+        :param name: The data identifier name.
+        :param meta: the metadata key-values.
+        :type meta: dict
+        :param recursive: Option to propagate the metadata change to content.
+        """
+        path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'meta'])
+        url = build_url(choice(self.list_hosts), path=path)
+        data = dumps({'meta': meta, 'recursive': recursive})
+        r = self._send_request(url, type='POST', data=data)
+        if r.status_code == codes.created:
+            return True
+        else:
+            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
+            raise exc_cls(exc_msg)
+
     def set_status(self, scope, name, **kwargs):
         """
         Set data identifier status

lib/rucio/core/did.py

@@ -1,4 +1,5 @@
-# Copyright 2013-2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2013-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,26 +14,26 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2019
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013-2018
-# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2015
-# - Wen Guan <wguan.icedew@gmail.com>, 2015
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2014-2015
+# - Wen Guan <wen.guan@cern.ch>, 2015
+# - asket <asket.agarwal96@gmail.com>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Tobias Wegner <twegner@cern.ch>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
+# - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2020
 # - Brandon White <bjwhite@fnal.gov>, 2019
 # - Luc Goossens <luc.goossens@cern.ch>, 2020
-# - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2019 - 2020
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - Vivek Nigam <viveknigam.nigam3@gmail.com>, 2020
-#
-# PY3K COMPATIBLE
 
 import logging
 import random
@@ -173,12 +174,13 @@ def add_dids(dids, account, session=None):
                 new_did = models.DataIdentifier(scope=did['scope'], name=did['name'], account=did.get('account') or account,
                                                 did_type=did['type'], monotonic=did.get('statuses', {}).get('monotonic', False),
                                                 is_open=True, expired_at=expired_at)
-                # Add metadata
-                for key in did.get('meta', {}):
-                    new_did.update({key: did['meta'][key]})
 
                 new_did.save(session=session, flush=False)
 
+                if 'meta' in did and did['meta']:
+                    # Add metadata
+                    set_metadata_bulk(scope=did['scope'], name=did['name'], meta=did['meta'], recursive=False, session=session)
+
                 if did.get('dids', None):
                     attach_dids(scope=did['scope'], name=did['name'], dids=did['dids'],
                                 account=account, rse_id=did.get('rse_id'), session=session)
@@ -1241,6 +1243,20 @@ def set_metadata(scope, name, key, value, type=None, did=None,
     did_meta_plugins.set_metadata(scope=scope, name=name, key=key, value=value, recursive=recursive, session=session)
 
 
+@transactional_session
+def set_metadata_bulk(scope, name, meta, recursive=False, session=None):
+    """
+    Add metadata to data identifier.
+
+    :param scope: The scope name.
+    :param name: The data identifier name.
+    :param meta: the key-values.
+    :param recursive: Option to propagate the metadata change to content.
+    :param session: The database session in use.
+    """
+    did_meta_plugins.set_metadata_bulk(scope=scope, name=name, meta=meta, recursive=recursive, session=session)
+
+
 @read_session
 def get_metadata(scope, name, plugin='DID_COLUMN', session=None):
     """

lib/rucio/core/did_meta_plugins/__init__.py

@@ -1,4 +1,5 @@
-# Copyright 2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,8 +16,6 @@
 # Authors:
 # - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
-#
-# PY3K COMPATIBLE
 
 import importlib
 
@@ -104,6 +103,32 @@ def set_metadata(scope, name, key, value, recursive=False, session=None):
             break
 
 
+def set_metadata_bulk(scope, name, meta, recursive=False, session=None):
+    """
+    Sets the metadata for a given did.
+
+    To decide which metadata store to use, it is checking the
+    configuration of the server and assigns each key-value to the
+    correct plugin by checking them in order of METADATA_HANDLERS.
+
+    :param scope: The scope name.
+    :param name: The data identifier name.
+    :param meta: all key-values to set.
+    :param recursive: Option to propagate the metadata change to content.
+    :param session: The database session in use.
+    """
+    remainder = dict(meta)
+    for meta_handler in METADATA_HANDLERS:
+        pluginmeta = {}
+        for key, value in remainder.items():
+            if meta_handler.manages_key(key):
+                pluginmeta[key] = value
+        if pluginmeta:
+            for key in pluginmeta:
+                del remainder[key]
+            meta_handler.set_metadata_bulk(scope, name, meta=pluginmeta, recursive=recursive, session=session)
+
+
 def delete_metadata(scope, name, key, session=None):
     """
     Deletes the metadata stored for the given key. Currently only works for JSON metadata store