From ea765b7e06565edfb0e4a4b9489b25b419490b7b Mon Sep 17 00:00:00 2001 From: voetberg Date: Wed, 21 Feb 2024 11:20:43 -0600 Subject: [PATCH] Metadata: Rename core.meta to core.meta_conventions Fix #5224 Changes: * Rename core.meta to core.meta_conventions * Rename DIDKey -> DIDMetaConventionsKey * Rename DIDKeyValue -> DIDMetaConventionsConstraints * Update docstrings to reflect naming, more descriptive * Typing on *.meta_conventions --- lib/rucio/api/did.py | 5 +- .../api/{meta.py => meta_conventions.py} | 28 +++++----- lib/rucio/client/client.py | 4 +- ...metaclient.py => metaconventionsclient.py} | 27 +++++----- .../core/{meta.py => meta_conventions.py} | 51 ++++++++++--------- lib/rucio/db/sqla/models.py | 16 +++--- lib/rucio/web/rest/flaskapi/v1/main.py | 11 +++- .../v1/{meta.py => meta_conventions.py} | 25 +++++++-- ...{test_meta.py => test_meta_conventions.py} | 8 +-- 9 files changed, 105 insertions(+), 70 deletions(-) rename lib/rucio/api/{meta.py => meta_conventions.py} (66%) rename lib/rucio/client/{metaclient.py => metaconventionsclient.py} (80%) rename lib/rucio/core/{meta.py => meta_conventions.py} (77%) rename lib/rucio/web/rest/flaskapi/v1/{meta.py => meta_conventions.py} (89%) rename tests/{test_meta.py => test_meta_conventions.py} (96%) diff --git a/lib/rucio/api/did.py b/lib/rucio/api/did.py index 39b5fb9d67..a9780ab08d 100644 --- a/lib/rucio/api/did.py +++ b/lib/rucio/api/did.py @@ -22,8 +22,9 @@ from rucio.common.schema import validate_schema from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import api_update_return_dict -from rucio.core import did, naming_convention, meta as meta_core +from rucio.core import did, naming_convention from rucio.core.rse import get_rse_id +from rucio.core import meta_conventions as meta_convention_core from rucio.db.sqla.constants import DIDType from rucio.db.sqla.session import read_session, stream_session, transactional_session @@ -117,7 +118,7 @@ def add_did(scope, name, did_type, issuer, account=None, statuses={}, meta={}, r raise rucio.common.exception.InvalidObject("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k])) # Validate metadata - meta_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session) + meta_convention_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session) return did.add_did(scope=scope, name=name, did_type=DIDType[did_type.upper()], account=account or issuer, statuses=statuses, meta=meta, rules=rules, lifetime=lifetime, diff --git a/lib/rucio/api/meta.py b/lib/rucio/api/meta_conventions.py similarity index 66% rename from lib/rucio/api/meta.py rename to lib/rucio/api/meta_conventions.py index b83a562b66..138c262272 100644 --- a/lib/rucio/api/meta.py +++ b/lib/rucio/api/meta_conventions.py @@ -13,33 +13,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Union, Optional from rucio.api.permission import has_permission from rucio.common.exception import AccessDenied -from rucio.core import meta +from rucio.core import meta_conventions from rucio.db.sqla.session import read_session, transactional_session +from rucio.db.sqla.constants import KeyType if TYPE_CHECKING: from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount @read_session def list_keys(*, session: "Session"): """ - Lists all keys. + Lists all keys for DID Metadata Conventions. :param session: The database session in use. :returns: A list containing all keys. """ - return meta.list_keys(session=session) + return meta_conventions.list_keys(session=session) @read_session -def list_values(key, *, session: "Session"): +def list_values(key: str, *, session: "Session"): """ - Lists all values for a key. + Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions). :param key: the name for the key. :param session: The database session in use. @@ -47,13 +49,13 @@ def list_values(key, *, session: "Session"): :returns: A list containing all values. """ - return meta.list_values(key=key, session=session) + return meta_conventions.list_values(key=key, session=session) @transactional_session -def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def', *, session: "Session"): +def add_key(key: str, key_type: Union[KeyType, str], issuer: "InternalAccount", value_type: Optional[str] = None, value_regexp: Optional[str] = None, vo: str = 'def', *, session: "Session"): """ - Add a new allowed key. + Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key). :param key: the name for the new key. :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection). @@ -66,13 +68,13 @@ def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def', kwargs = {'key': key, 'key_type': key_type, 'value_type': value_type, 'value_regexp': value_regexp} if not has_permission(issuer=issuer, vo=vo, action='add_key', kwargs=kwargs, session=session): raise AccessDenied('Account %s can not add key' % (issuer)) - return meta.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session) + return meta_conventions.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session) @transactional_session -def add_value(key, value, issuer, vo='def', *, session: "Session"): +def add_value(key: str, value: str, issuer: "InternalAccount", vo: str = 'def', *, session: "Session"): """ - Add a new value to a key. + Add an allowed value for DID metadata (update a key in DID Metadata Conventions table). :param key: the name for the key. :param value: the value. @@ -82,4 +84,4 @@ def add_value(key, value, issuer, vo='def', *, session: "Session"): kwargs = {'key': key, 'value': value} if not has_permission(issuer=issuer, vo=vo, action='add_value', kwargs=kwargs, session=session): raise AccessDenied('Account %s can not add value %s to key %s' % (issuer, value, key)) - return meta.add_value(key=key, value=value, session=session) + return meta_conventions.add_value(key=key, value=value, session=session) diff --git a/lib/rucio/client/client.py b/lib/rucio/client/client.py index ac16dbd4d1..af908b51e6 100644 --- a/lib/rucio/client/client.py +++ b/lib/rucio/client/client.py @@ -27,7 +27,7 @@ from rucio.client.importclient import ImportClient from rucio.client.lifetimeclient import LifetimeClient from rucio.client.lockclient import LockClient -from rucio.client.metaclient import MetaClient +from rucio.client.metaconventionsclient import MetaConventionClient from rucio.client.pingclient import PingClient from rucio.client.replicaclient import ReplicaClient from rucio.client.requestclient import RequestClient @@ -40,7 +40,7 @@ class Client(AccountClient, AccountLimitClient, - MetaClient, + MetaConventionClient, PingClient, ReplicaClient, RequestClient, diff --git a/lib/rucio/client/metaclient.py b/lib/rucio/client/metaconventionsclient.py similarity index 80% rename from lib/rucio/client/metaclient.py rename to lib/rucio/client/metaconventionsclient.py index 6b9b779d6d..8959771b49 100644 --- a/lib/rucio/client/metaclient.py +++ b/lib/rucio/client/metaconventionsclient.py @@ -21,17 +21,19 @@ from rucio.client.baseclient import BaseClient from rucio.client.baseclient import choice from rucio.common.utils import build_url +from rucio.db.sqla.constants import KeyType +from typing import Union, Optional -class MetaClient(BaseClient): +class MetaConventionClient(BaseClient): - """Meta client class for working with data identifier attributes""" + """Metadata client class for working with data identifier attributes""" - META_BASEURL = 'meta' + META_BASEURL = 'meta_conventions' - def add_key(self, key, key_type, value_type=None, value_regexp=None): + def add_key(self, key: str, key_type: Union[KeyType, str], value_type: Optional[str] = None, value_regexp: Optional[str] = None) -> Optional[bool]: """ - Sends the request to add a new key. + Sends the request to add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key). :param key: the name for the new key. :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection). @@ -56,9 +58,9 @@ def add_key(self, key, key_type, value_type=None, value_regexp=None): exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg) - def list_keys(self): + def list_keys(self) -> Optional[list[str]]: """ - Sends the request to list all keys. + Sends the request to list all keys for DID Metadata Conventions. :return: a list containing the names of all keys. """ @@ -72,9 +74,10 @@ def list_keys(self): exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg) - def list_values(self, key): + def list_values(self, key: str) -> Optional[list[str]]: """ - Sends the request to list all values for a key. + Sends the request to lists all allowed values for a DID key (all values for a key in DID Metadata Conventions). +. :return: a list containing the names of all values for a key. """ @@ -88,9 +91,9 @@ def list_values(self, key): exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg) - def add_value(self, key, value): + def add_value(self, key: str, value: str) -> Optional[bool]: """ - Sends the request to add a value to a key. + Sends the request to add a value for a key in DID Metadata Convention. :param key: the name for key. :param value: the value. @@ -111,7 +114,7 @@ def add_value(self, key, value): def del_value(self, key, value): """ - Delete a value for a key. + Delete a key in the DID Metadata Conventions table. :param key: the name for key. :param value: the value. diff --git a/lib/rucio/core/meta.py b/lib/rucio/core/meta_conventions.py similarity index 77% rename from lib/rucio/core/meta.py rename to lib/rucio/core/meta_conventions.py index 8ec9225310..e6556fae3d 100644 --- a/lib/rucio/core/meta.py +++ b/lib/rucio/core/meta_conventions.py @@ -14,10 +14,9 @@ # limitations under the License. from re import match -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional, Union -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.exc import IntegrityError, NoResultFound from rucio.common.constraints import AUTHORIZED_VALUE_TYPES from rucio.common.exception import (Duplicate, RucioException, @@ -32,9 +31,9 @@ @transactional_session -def add_key(key, key_type, value_type=None, value_regexp=None, *, session: "Session"): +def add_key(key: str, key_type: Union[KeyType, str], value_type: Optional[str] = None, value_regexp: Optional[str] = None, *, session: "Session") -> None: """ - Adds a new allowed key. + Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key). :param key: the name for the new key. :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection). @@ -65,7 +64,7 @@ def add_key(key, key_type, value_type=None, value_regexp=None, *, session: "Sess except ValueError: raise UnsupportedKeyType('The type \'%s\' is not supported for keys!' % str(key_type)) - new_key = models.DIDKey(key=key, value_type=value_type and str(value_type), value_regexp=value_regexp, key_type=key_type) + new_key = models.DIDMetaConventionsKey(key=key, value_type=value_type and str(value_type), value_regexp=value_regexp, key_type=key_type) try: new_key.save(session=session) except IntegrityError as error: @@ -77,46 +76,50 @@ def add_key(key, key_type, value_type=None, value_regexp=None, *, session: "Sess or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \ or match('.*IntegrityError.*columns? key.*not unique.*', error.args[0]): raise Duplicate(f"key '{key}' already exists!") - raise + raise RucioException(error.args) @transactional_session -def del_key(key, *, session: "Session"): +def del_key(key: str, *, session: "Session") -> None: """ - Deletes a key. + Delete a key in the DID Metadata Conventions table. :param key: the name for the key. :param session: The database session in use. """ - session.query(models.DIDKey).filter(key == key).delete() + session.query(models.DIDMetaConventionsKey).filter(key == key).delete() @read_session -def list_keys(*, session: "Session"): +def list_keys(*, session: "Session") -> list[str]: """ - Lists all keys. + Lists all keys for DID Metadata Conventions. :param session: The database session in use. :returns: A list containing all keys. """ key_list = [] - query = session.query(models.DIDKey) + query = session.query(models.DIDMetaConventionsKey) for row in query: key_list.append(row.key) return key_list @transactional_session -def add_value(key, value, *, session: "Session"): +def add_value(key: str, value: str, *, session: "Session") -> None: """ - Adds a new value to a key. + Adds a new value for a key in DID Metadata Convention. :param key: the name for the key. :param value: the value. :param session: The database session in use. + + :raises Duplicate: Key-Value pair exists + :raises KeyNotFound: Key not in metadata conventions table + :raises InvalidValueForKey: Value conflicts with rse expression for key values or does not have the correct type """ - new_value = models.DIDKeyValueAssociation(key=key, value=value) + new_value = models.DIDMetaConventionsConstraints(key=key, value=value) try: new_value.save(session=session) except IntegrityError as error: @@ -137,7 +140,7 @@ def add_value(key, value, *, session: "Session"): raise RucioException(error.args) - k = session.query(models.DIDKey).filter_by(key=key).one() + k = session.query(models.DIDMetaConventionsKey).filter_by(key=key).one() # Check value against regexp, if defined if k.value_regexp and not match(k.value_regexp, value): @@ -145,14 +148,14 @@ def add_value(key, value, *, session: "Session"): # Check value type, if defined type_map = dict([(str(t), t) for t in AUTHORIZED_VALUE_TYPES]) - if k.value_type and not isinstance(value, type_map.get(k.value_type)): + if k.value_type and not isinstance(value, type_map.get(k.value_type)): # type: ignore ; Typing error caused by 'isinstaince' not thinking types count as classes raise InvalidValueForKey("The value '%s' for the key '%s' does not match the required type '%s'" % (value, key, k.value_type)) @read_session -def list_values(key, *, session: "Session"): +def list_values(key: str, *, session: "Session") -> list[str]: """ - Lists all values for a key. + Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions). :param key: the name for the key. :param session: The database session in use. @@ -160,14 +163,14 @@ def list_values(key, *, session: "Session"): :returns: A list containing all values. """ value_list = [] - query = session.query(models.DIDKeyValueAssociation).filter_by(key=key) + query = session.query(models.DIDMetaConventionsConstraints).filter_by(key=key) for row in query: value_list.append(row.value) return value_list @read_session -def validate_meta(meta, did_type, *, session: "Session"): +def validate_meta(meta: dict, did_type: DIDType, *, session: "Session") -> None: """ Validates metadata for a did. @@ -175,13 +178,13 @@ def validate_meta(meta, did_type, *, session: "Session"): :param meta: the type of the did, e.g, DATASET, CONTAINER, FILE. :param session: The database session in use. - :returns: True + :raises InvalidObject: """ # For now only validate the datatype for datasets key = 'datatype' if did_type == DIDType.DATASET and key in meta: try: - session.query(models.DIDKeyValueAssociation.value).\ + session.query(models.DIDMetaConventionsConstraints.value).\ filter_by(key=key).\ filter_by(value=meta[key]).\ one() diff --git a/lib/rucio/db/sqla/models.py b/lib/rucio/db/sqla/models.py index 7887eac146..a11a0570ee 100644 --- a/lib/rucio/db/sqla/models.py +++ b/lib/rucio/db/sqla/models.py @@ -605,8 +605,8 @@ class QuarantinedReplicaHistory(BASE, ModelBase): _table_args = () -class DIDKey(BASE, ModelBase): - """Represents Data IDentifier property keys""" +class DIDMetaConventionsKey(BASE, ModelBase): + """Represents allowed keys of DID Metadata""" __tablename__ = 'did_keys' key: Mapped[str] = mapped_column(String(255)) is_enum: Mapped[bool] = mapped_column(Boolean(name='DID_KEYS_IS_ENUM_CHK', create_constraint=True), @@ -621,8 +621,8 @@ class DIDKey(BASE, ModelBase): CheckConstraint('is_enum IS NOT NULL', name='DID_KEYS_IS_ENUM_NN')) -class DIDKeyValueAssociation(BASE, ModelBase): - """Represents Data IDentifier property key/values""" +class DIDMetaConventionsConstraints(BASE, ModelBase): + """Represents a map for constraint values a DID metadata key must follow """ __tablename__ = 'did_key_map' key: Mapped[str] = mapped_column(String(255)) value: Mapped[str] = mapped_column(String(255)) @@ -1696,8 +1696,8 @@ def register_models(engine): ConstituentAssociationHistory, DataIdentifierAssociation, DataIdentifierAssociationHistory, - DIDKey, - DIDKeyValueAssociation, + DIDMetaConventionsKey, + DIDMetaConventionsConstraints, DataIdentifier, DidMeta, VirtualPlacements, @@ -1766,8 +1766,8 @@ def unregister_models(engine): ConstituentAssociationHistory, DataIdentifierAssociation, DataIdentifierAssociationHistory, - DIDKey, - DIDKeyValueAssociation, + DIDMetaConventionsKey, + DIDMetaConventionsConstraints, DidMeta, DataIdentifier, DeletedDataIdentifier, diff --git a/lib/rucio/web/rest/flaskapi/v1/main.py b/lib/rucio/web/rest/flaskapi/v1/main.py index d88c79307b..b968010fee 100644 --- a/lib/rucio/web/rest/flaskapi/v1/main.py +++ b/lib/rucio/web/rest/flaskapi/v1/main.py @@ -21,6 +21,7 @@ from rucio.common.config import config_get from rucio.common.exception import ConfigurationError from rucio.common.logging import setup_logging +import logging from rucio.web.rest.flaskapi.v1.common import CORSMiddleware DEFAULT_ENDPOINTS = [ @@ -37,7 +38,7 @@ 'import', 'lifetime_exceptions', 'locks', - 'meta', + 'meta_conventions', 'ping', 'redirect', 'replicas', @@ -51,6 +52,10 @@ def apply_endpoints(app, modules): for blueprint_module in modules: + # Legacy patch - TODO Remove in 38.0.0 + if blueprint_module == "meta": + logging.log(logging.WARNING, "Endpoint `meta` is depreciated and will be removed in future releaases") + blueprint_module = "meta_conventions" try: # searches for module names locally blueprint_module = importlib.import_module('.' + blueprint_module, @@ -60,6 +65,10 @@ def apply_endpoints(app, modules): if hasattr(blueprint_module, 'blueprint'): app.register_blueprint(blueprint_module.blueprint()) + + if hasattr(blueprint_module, "blueprint_legacy"): + app.register_blueprint(blueprint_module.blueprint_legacy()) + else: raise ConfigurationError(f'"{blueprint_module}" from the endpoints configuration value did not have a blueprint') diff --git a/lib/rucio/web/rest/flaskapi/v1/meta.py b/lib/rucio/web/rest/flaskapi/v1/meta_conventions.py similarity index 89% rename from lib/rucio/web/rest/flaskapi/v1/meta.py rename to lib/rucio/web/rest/flaskapi/v1/meta_conventions.py index b5c99004fb..9e1c740db8 100644 --- a/lib/rucio/web/rest/flaskapi/v1/meta.py +++ b/lib/rucio/web/rest/flaskapi/v1/meta_conventions.py @@ -15,15 +15,15 @@ from flask import Flask, request, jsonify -from rucio.api.meta import add_key, add_value, list_keys, list_values +from rucio.api.meta_conventions import add_key, add_value, list_keys, list_values from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedValueType, UnsupportedKeyType from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, response_headers, \ generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get -class Meta(ErrorHandlingMethodView): - """ REST APIs for data identifier attribute keys. """ +class MetaConventions(ErrorHandlingMethodView): + """ REST APIs for managing data identifier attribute metadata key formats. """ @check_accept_header_wrapper_flask(['application/json']) def get(self): @@ -207,9 +207,23 @@ def post(self, key): def blueprint(): + bp = AuthenticatedBlueprint('meta_conventions', __name__, url_prefix='/meta_conventions') + + meta_view = MetaConventions.as_view('meta_conventions') + bp.add_url_rule('/', view_func=meta_view, methods=['get', ]) + bp.add_url_rule('/', view_func=meta_view, methods=['post', ]) + values_view = Values.as_view('values') + bp.add_url_rule('//', view_func=values_view, methods=['get', 'post']) + + bp.after_request(response_headers) + return bp + + +def blueprint_legacy(): + # TODO: Remove in 38.0 bp = AuthenticatedBlueprint('meta', __name__, url_prefix='/meta') - meta_view = Meta.as_view('meta') + meta_view = MetaConventions.as_view('meta') bp.add_url_rule('/', view_func=meta_view, methods=['get', ]) bp.add_url_rule('/', view_func=meta_view, methods=['post', ]) values_view = Values.as_view('values') @@ -222,5 +236,8 @@ def blueprint(): def make_doc(): """ Only used for sphinx documentation """ doc_app = Flask(__name__) + doc_app.register_blueprint(blueprint()) + doc_app.register_blueprint(blueprint_legacy()) + return doc_app diff --git a/tests/test_meta.py b/tests/test_meta_conventions.py similarity index 96% rename from tests/test_meta.py rename to tests/test_meta_conventions.py index b4a31ca021..25964ac0a3 100644 --- a/tests/test_meta.py +++ b/tests/test_meta_conventions.py @@ -17,13 +17,13 @@ from rucio.common.exception import InvalidValueForKey, RucioException, UnsupportedValueType, UnsupportedKeyType from rucio.common.utils import generate_uuid as uuid -from rucio.core.meta import add_key +from rucio.core.meta_conventions import add_key from rucio.db.sqla import session, models from rucio.db.sqla.constants import DIDType, KeyType @pytest.mark.dirty -class TestMetaClient: +class TestMetaConventionsClient: def test_add_and_list_keys(self, rucio_client): """ META (CLIENTS): Add a key and List all keys.""" @@ -99,7 +99,7 @@ def test_add_key(self, rucio_client): for key_type in types: key_name = 'datatype%s' % str(uuid()) rucio_client.add_key(key_name, key_type['type']) - stored_key_type = session.get_session().query(models.DIDKey).filter_by(key=key_name).one()['key_type'] + stored_key_type = session.get_session().query(models.DIDMetaConventionsKey).filter_by(key=key_name).one()['key_type'] assert stored_key_type, key_type['expected'] with pytest.raises(UnsupportedKeyType): @@ -131,7 +131,7 @@ def test_add_key(): for key_type in types: key_name = 'datatype%s' % str(uuid()) add_key(key_name, key_type['type']) - stored_key_type = session.get_session().query(models.DIDKey).filter_by(key=key_name).one()['key_type'] + stored_key_type = session.get_session().query(models.DIDMetaConventionsKey).filter_by(key=key_name).one()['key_type'] assert stored_key_type, key_type['expected'] with pytest.raises(UnsupportedKeyType):