Core & Internals: Add setting did meta in bulk; rucio#3847

lib/rucio/api/did.py

@@ -421,6 +421,30 @@ def set_metadata(scope, name, key, value, issuer, recursive=False, vo='def'):
     return did.set_metadata(scope=scope, name=name, key=key, value=value, recursive=recursive)
 
 
+def set_metadata_bulk(scope, name, meta, issuer, recursive=False, vo='def'):
+    """
+    Add metadata to data did.
+
+    :param scope: The scope name.
+    :param name: The data identifier name.
+    :param meta: the key-values.
+    :param issuer: The issuer account.
+    :param recursive: Option to propagate the metadata update to content.
+    :param vo: The VO to act on.
+    """
+    kwargs = {'scope': scope, 'name': name, 'meta': meta, 'issuer': issuer}
+
+    for key in meta:
+        if key in RESERVED_KEYS:
+            raise rucio.common.exception.AccessDenied('Account %s can not change the value of the metadata key %s to data identifier %s:%s' % (issuer, key, scope, name))
+
+    if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='set_metadata_bulk', kwargs=kwargs):
+        raise rucio.common.exception.AccessDenied('Account %s can not add metadata to data identifier %s:%s' % (issuer, scope, name))
+
+    scope = InternalScope(scope, vo=vo)
+    return did.set_metadata_bulk(scope=scope, name=name, meta=meta, recursive=recursive)
+
+
 def get_metadata(scope, name, plugin='DID_COLUMN', vo='def'):
     """
     Get data identifier metadata

lib/rucio/client/didclient.py

@@ -477,6 +477,26 @@ def set_metadata(self, scope, name, key, value, recursive=False):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
+    def set_metadata_bulk(self, scope, name, meta, recursive=False):
+        """
+        Set data identifier metadata in bulk.
+
+        :param scope: The scope name.
+        :param name: The data identifier name.
+        :param meta: the metadata key-values.
+        :type meta: dict
+        :param recursive: Option to propagate the metadata change to content.
+        """
+        path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'meta'])
+        url = build_url(choice(self.list_hosts), path=path)
+        data = dumps({'meta': meta, 'recursive': recursive})
+        r = self._send_request(url, type='POST', data=data)
+        if r.status_code == codes.created:
+            return True
+        else:
+            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
+            raise exc_cls(exc_msg)
+
     def set_status(self, scope, name, **kwargs):
         """
         Set data identifier status

lib/rucio/core/did.py

@@ -1241,6 +1241,20 @@ def set_metadata(scope, name, key, value, type=None, did=None,
     did_meta_plugins.set_metadata(scope=scope, name=name, key=key, value=value, recursive=recursive, session=session)
 
 
+@transactional_session
+def set_metadata_bulk(scope, name, meta, recursive=False, session=None):
+    """
+    Add metadata to data identifier.
+
+    :param scope: The scope name.
+    :param name: The data identifier name.
+    :param meta: the key-values.
+    :param recursive: Option to propagate the metadata change to content.
+    :param session: The database session in use.
+    """
+    did_meta_plugins.set_metadata_bulk(scope=scope, name=name, meta=meta, recursive=recursive, session=session)
+
+
 @read_session
 def get_metadata(scope, name, plugin='DID_COLUMN', session=None):
     """

lib/rucio/core/did_meta_plugins/__init__.py

@@ -104,6 +104,32 @@ def set_metadata(scope, name, key, value, recursive=False, session=None):
             break
 
 
+def set_metadata_bulk(scope, name, meta, recursive=False, session=None):
+    """
+    Sets the metadata for a given did.
+
+    To decide which metadata store to use, it is checking the
+    configuration of the server and assigns each key-value to the
+    correct plugin by checking them in order of METADATA_HANDLERS.
+
+    :param scope: The scope name.
+    :param name: The data identifier name.
+    :param meta: all key-values to set.
+    :param recursive: Option to propagate the metadata change to content.
+    :param session: The database session in use.
+    """
+    remainder = dict(meta)
+    for meta_handler in METADATA_HANDLERS:
+        pluginmeta = {}
+        for key, value in remainder.items():
+            if meta_handler.manages_key(key):
+                pluginmeta[key] = value
+        if pluginmeta:
+            for key in pluginmeta:
+                del remainder[key]
+            meta_handler.set_metadata_bulk(scope, name, meta=pluginmeta, recursive=recursive, session=session)
+
+
 def delete_metadata(scope, name, key, session=None):
     """
     Deletes the metadata stored for the given key. Currently only works for JSON metadata store

lib/rucio/core/did_meta_plugins/did_meta_plugin_interface.py

@@ -40,6 +40,20 @@ def set_metadata(self, scope, name, key, value, recursive=False, session=None):
         """
         pass
 
+    def set_metadata_bulk(self, scope, name, meta, recursive=False, session=None):
+        """
+        Add metadata to data identifier in bulk.
+
+        :param scope: The scope name.
+        :param name: The data identifier name.
+        :param meta: all key-values to set.
+        :type meta: dict
+        :param recursive: Option to propagate the metadata change to content.
+        :param session: The database session in use.
+        """
+        for key, value in meta.items():
+            self.set_metadata(scope, name, key, value, recursive=recursive, session=session)
+
     @abstractmethod
     def delete_metadata(self, scope, name, key, session=None):
         """

lib/rucio/web/rest/flaskapi/v1/did.py

@@ -39,7 +39,7 @@
                            list_dids, list_dids_extended, list_files, scope_list, get_did,
                            set_metadata, get_metadata, get_metadata_bulk, set_status, attach_dids, detach_dids,
                            attach_dids_to_dids, get_dataset_by_guid, list_parent_dids,
-                           create_did_sample, list_new_dids, resurrect)
+                           create_did_sample, list_new_dids, resurrect, set_metadata_bulk)
 from rucio.api.rule import list_replication_rules, list_associated_replication_rules_for_file
 from rucio.common.exception import (ScopeNotFound, DataIdentifierNotFound,
                                     DataIdentifierAlreadyExists, DuplicateContent,
@@ -851,11 +851,60 @@ def get(self, scope, name):
             print(format_exc())
             return error, 500
 
+    def post(self, scope, name):
+        """
+        Add metadata to a data identifier in bulk.
+
+        .. :quickref: Meta; Add DID metadata.
+
+        HTTP Success:
+            201 Created
+
+        HTTP Error:
+            400 Bad Request
+            401 Unauthorized
+            404 Not Found
+            409 Conflict
+            500 Internal Error
+
+        :param scope: The scope name.
+        :param name: The data identifier name.
+        """
+        json_data = request.data
+        try:
+            params = loads(json_data)
+            meta = params['meta']
+            recursive = params.get('recursive', False)
+        except ValueError:
+            return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')
+        try:
+            set_metadata_bulk(scope=scope, name=name, meta=meta,
+                              issuer=request.environ.get('issuer'), recursive=recursive, vo=request.environ.get('vo'))
+        except DataIdentifierNotFound as error:
+            return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
+        except Duplicate as error:
+            return generate_http_error_flask(409, 'Duplicate', error.args[0])
+        except KeyNotFound as error:
+            return generate_http_error_flask(400, 'KeyNotFound', error.args[0])
+        except InvalidMetadata as error:
+            return generate_http_error_flask(400, 'InvalidMetadata', error.args[0])
+        except InvalidValueForKey as error:
+            return generate_http_error_flask(400, 'InvalidValueForKey', error.args[0])
+        except RucioException as error:
+            return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
+        except Exception as error:
+            print(format_exc())
+            return error, 500
+
+        return "Created", 201
+
+
+class SingleMeta(MethodView):
     def post(self, scope, name, key):
         """
         Add metadata to a data identifier.
 
-        .. :quickref: Meta; Add DID metadata.
+        .. :quickref: SingleMeta; Add DID metadata.
 
         HTTP Success:
             201 Created
@@ -1183,8 +1232,9 @@ def post(self):
 attachment_view = Attachment.as_view('attachment')
 bp.add_url_rule('/<scope>/<name>/dids', view_func=attachment_view, methods=['get', 'post', 'delete'])
 meta_view = Meta.as_view('meta')
-bp.add_url_rule('/<scope>/<name>/meta', view_func=meta_view, methods=['get', ])
-bp.add_url_rule('/<scope>/<name>/meta/<key>', view_func=meta_view, methods=['post', ])
+bp.add_url_rule('/<scope>/<name>/meta', view_func=meta_view, methods=['get', 'post'])
+singlemeta_view = SingleMeta.as_view('meta')
+bp.add_url_rule('/<scope>/<name>/meta/<key>', view_func=singlemeta_view, methods=['post', ])
 rules_view = Rules.as_view('rules')
 bp.add_url_rule('/<scope>/<name>/rules', view_func=rules_view, methods=['get', ])
 parents_view = Parents.as_view('parents')

lib/rucio/web/rest/webpy/v1/did.py

@@ -33,20 +33,18 @@
 # PY3K COMPATIBLE
 
 from __future__ import print_function
+
 from json import dumps, loads
 from traceback import format_exc
-try:
-    from urlparse import parse_qs
-except ImportError:
-    from urllib.parse import parse_qs
+
 from web import application, ctx, data, Created, header, InternalError, OK, loadhook
 
 from rucio.api.did import (add_did, add_dids, list_content, list_content_history,
                            list_dids, list_dids_extended, list_files, scope_list, get_did, set_metadata,
                            get_metadata, get_metadata_bulk, delete_metadata, set_status, attach_dids, detach_dids,
                            attach_dids_to_dids, get_dataset_by_guid, list_parent_dids,
                            create_did_sample, list_new_dids, resurrect, add_did_to_followed,
-                           get_users_following_did, remove_did_from_followed)
+                           get_users_following_did, remove_did_from_followed, set_metadata_bulk)
 from rucio.api.rule import list_replication_rules, list_associated_replication_rules_for_file
 from rucio.common.exception import (ScopeNotFound, DataIdentifierNotFound,
                                     DataIdentifierAlreadyExists, DuplicateContent,
@@ -59,6 +57,11 @@
 from rucio.common.utils import generate_http_error, render_json, APIEncoder, parse_response
 from rucio.web.rest.common import rucio_loadhook, RucioController, check_accept_header_wrapper
 
+try:
+    from urlparse import parse_qs
+except ImportError:
+    from urllib.parse import parse_qs
+
 URLS = (
     '/(.*)/$', 'Scope',
     '/(.*)/guid', 'GUIDLookup',
@@ -67,13 +70,12 @@
     '%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'Files',
     '%s/dids/history' % get_schema_value('SCOPE_NAME_REGEXP'), 'AttachmentHistory',
     '%s/dids' % get_schema_value('SCOPE_NAME_REGEXP'), 'Attachment',
-    '%s/meta/(.*)' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta',
+    '%s/meta/(.*)' % get_schema_value('SCOPE_NAME_REGEXP'), 'SingleMeta',
     '%s/meta' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta',
     '%s/status' % get_schema_value('SCOPE_NAME_REGEXP'), 'DIDs',
     '%s/rules' % get_schema_value('SCOPE_NAME_REGEXP'), 'Rules',
     '%s/parents' % get_schema_value('SCOPE_NAME_REGEXP'), 'Parents',
     '%s/associated_rules' % get_schema_value('SCOPE_NAME_REGEXP'), 'AssociatedRules',
-    '%s/did_meta' % get_schema_value('SCOPE_NAME_REGEXP'), 'DidMeta',
     '/(.*)/(.*)/(.*)/(.*)/(.*)/sample', 'Sample',
     '%s' % get_schema_value('SCOPE_NAME_REGEXP'), 'DIDs',
     '', 'BulkDIDS',
@@ -669,9 +671,11 @@ def GET(self, scope, name):
             print(format_exc())
             raise InternalError(error)
 
-    def POST(self, scope, name, key):
+    def POST(self, scope, name):
         """
-        Add metadata to a data identifier.
+        Add metadata to a data identifier in bulk.
+
+        .. :quickref: Meta; Add DID metadata.
 
         HTTP Success:
             201 Created
@@ -685,19 +689,17 @@ def POST(self, scope, name, key):
 
         :param scope: The scope name.
         :param name: The data identifier name.
-        :param key: the key.
-
         """
         json_data = data()
         try:
             params = loads(json_data)
-            value = params['value']
+            meta = params['meta']
             recursive = params.get('recursive', False)
         except ValueError:
             raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
         try:
-            set_metadata(scope=scope, name=name, key=key, value=value,
-                         issuer=ctx.env.get('issuer'), recursive=recursive, vo=ctx.env.get('vo'))
+            set_metadata_bulk(scope=scope, name=name, meta=meta,
+                              issuer=ctx.env.get('issuer'), recursive=recursive, vo=ctx.env.get('vo'))
         except DataIdentifierNotFound as error:
             raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
         except Duplicate as error:
@@ -712,9 +714,9 @@ def POST(self, scope, name, key):
             raise generate_http_error(500, error.__class__.__name__, error.args[0])
         except Exception as error:
             print(format_exc())
-            raise InternalError(error)
+            return error, 500
 
-        raise Created()
+        return "Created", 201
 
     def DELETE(self, scope, name):
         """
@@ -750,6 +752,56 @@ def DELETE(self, scope, name):
         raise OK()
 
 
+class SingleMeta(RucioController):
+
+    def POST(self, scope, name, key):
+        """
+        Add metadata to a data identifier.
+
+        HTTP Success:
+            201 Created
+
+        HTTP Error:
+            400 Bad Request
+            401 Unauthorized
+            404 Not Found
+            409 Conflict
+            500 Internal Error
+
+        :param scope: The scope name.
+        :param name: The data identifier name.
+        :param key: the key.
+
+        """
+        json_data = data()
+        try:
+            params = loads(json_data)
+            value = params['value']
+            recursive = params.get('recursive', False)
+        except ValueError:
+            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
+        try:
+            set_metadata(scope=scope, name=name, key=key, value=value,
+                         issuer=ctx.env.get('issuer'), recursive=recursive, vo=ctx.env.get('vo'))
+        except DataIdentifierNotFound as error:
+            raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
+        except Duplicate as error:
+            raise generate_http_error(409, 'Duplicate', error.args[0])
+        except KeyNotFound as error:
+            raise generate_http_error(400, 'KeyNotFound', error.args[0])
+        except InvalidMetadata as error:
+            raise generate_http_error(400, 'InvalidMetadata', error.args[0])
+        except InvalidValueForKey as error:
+            raise generate_http_error(400, 'InvalidValueForKey', error.args[0])
+        except RucioException as error:
+            raise generate_http_error(500, error.__class__.__name__, error.args[0])
+        except Exception as error:
+            print(format_exc())
+            raise InternalError(error)
+
+        raise Created()
+
+
 class Rules(RucioController):
 
     @check_accept_header_wrapper(['application/x-json-stream'])