From 2e78fd79f3d46912b68dc105e3c97fbaf3a8f8ba Mon Sep 17 00:00:00 2001 From: voetberg Date: Thu, 18 Jan 2024 09:06:08 -0600 Subject: [PATCH] Client: Allow client to initialize without a set config file #6410 --- lib/rucio/client/baseclient.py | 15 +- lib/rucio/common/config.py | 32 ++++- tests/test_clients.py | 245 ++++++++++++++++++++++++++++----- 3 files changed, 245 insertions(+), 47 deletions(-) diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py index 66faea8e77..6540d2f047 100644 --- a/lib/rucio/client/baseclient.py +++ b/lib/rucio/client/baseclient.py @@ -111,8 +111,10 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, self.host = config_get('client', 'rucio_host') if self.auth_host is None: self.auth_host = config_get('client', 'auth_host') - except (NoOptionError, NoSectionError) as error: - raise MissingClientParameter('Section client and Option \'%s\' cannot be found in config file' % error.args[0]) + except (NoOptionError, NoSectionError): + self.host = "https://rucio:443" + self.auth_host = "https://rucio:443" + self.logger.debug(f"Section client and Option rucio_host, auth_host cannot be found in config file, using {self.host} as host") try: self.trace_host = config_get('trace', 'trace_host') @@ -145,8 +147,9 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, else: try: self.auth_type = config_get('client', 'auth_type') - except (NoOptionError, NoSectionError) as error: - raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0]) + except (NoOptionError, NoSectionError): + self.auth_type = "userpass" + self.logger.debug(f'Option client/auth_type cannot be found in config file, using {self.auth_type}.') if self.auth_type == 'oidc': if not self.creds: @@ -282,9 +285,7 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, self.__authenticate() try: - self.request_retries = config_get_int('client', 'request_retries') - except (NoOptionError, RuntimeError): - LOG.debug('request_retries not specified in config file. Taking default.') + self.request_retries = config_get_int('client', 'request_retries', raise_exception=False, default=self.request_retries) except ValueError: self.logger.debug('request_retries must be an integer. Taking default.') diff --git a/lib/rucio/common/config.py b/lib/rucio/common/config.py index 1732d5e103..e242cca200 100644 --- a/lib/rucio/common/config.py +++ b/lib/rucio/common/config.py @@ -19,17 +19,22 @@ import json import os from collections.abc import Callable -from typing import TYPE_CHECKING, overload, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Any, overload, Optional, TypeVar, Union from rucio.common import exception from rucio.common.exception import ConfigNotFound, DatabaseException +import logging +import tempfile + _T = TypeVar('_T') _U = TypeVar('_U') if TYPE_CHECKING: from sqlalchemy.orm import Session + LoggerFunction = Callable[..., Any] + def convert_to_any_type(value) -> Union[bool, int, float, str]: if value.lower() in ['true', 'yes', 'on']: @@ -777,7 +782,7 @@ class Config: The configuration class reading the config file on init, located by using get_config_dirs or the use of the RUCIO_CONFIG environment variable. """ - def __init__(self): + def __init__(self, logger: "LoggerFunction" = logging.log): self.parser = configparser.ConfigParser() if 'RUCIO_CONFIG' in os.environ: @@ -786,9 +791,26 @@ def __init__(self): configs = [os.path.join(confdir, 'rucio.cfg') for confdir in get_config_dirs()] self.configfile = next(iter(filter(os.path.exists, configs)), None) if self.configfile is None: - raise RuntimeError('Could not load Rucio configuration file. ' - 'Rucio looked in the following paths for a configuration file, in order:' - '\n\t' + '\n\t'.join(configs)) + if ('RUCIO_CLIENT_MODE' not in os.environ) or (not os.environ['RUCIO_CLIENT_MODE']): + raise RuntimeError( + 'Could not load configuration file. ' + 'A configuration file is required to run in server mode. ' + 'If trying to run in client mode, be sure to set RUCIO_CLIENT_MODE=True in your envoriment. ' + 'Rucio looked in the following paths for a configuration file, in order: ' + '\n\t' + '\n\t'.join(configs)) + + logger( + level=30, + msg='Could not load Rucio configuration file. ' + 'Rucio looked in the following paths for a configuration file, in order: ' + '\n\t' + '\n\t'.join(configs) + '' + '\n\t Using empty configuration.') + + # Make a temp cfg file + self.configfile = tempfile.NamedTemporaryFile(suffix=".cfg").name + with open(self.configfile, 'w') as f: + f.write("") # File must have some content + f.close() if not self.parser.read(self.configfile) == [self.configfile]: raise RuntimeError('Could not load Rucio configuration file. ' diff --git a/tests/test_clients.py b/tests/test_clients.py index 8aad23db46..db567ac71d 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -15,15 +15,18 @@ from datetime import datetime, timedelta -from os import rename +import os +from functools import wraps +from random import choice +from string import ascii_lowercase import pytest -from rucio.client.baseclient import BaseClient -from rucio.client.client import Client from rucio.common.config import config_get, config_set, Config -from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException +from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException, RequestNotFound from rucio.common.utils import execute +from rucio.common.utils import setup_logger +from rucio.tests.common import skip_non_atlas from tests.mocks.mock_http_server import MockServer @@ -35,6 +38,20 @@ def client_token_path_override(file_config_mock, function_scope_prefix, tmp_path config_set('client', 'auth_token_file_path', str(tmp_path / f'{function_scope_prefix}token')) +def run_without_config(func): + @wraps(func) + def wrapper(*args, **kwargs): + configfile = Config().configfile + os.environ['RUCIO_CLIENT_MODE'] = 'True' + os.rename(configfile, f"{configfile}.tmp") + + try: + func(*args, **kwargs) + finally: + os.rename(f"{configfile}.tmp", configfile) + return wrapper + + @pytest.mark.usefixtures("client_token_path_override") class TestBaseClient: """ To test Clients""" @@ -44,42 +61,49 @@ class TestBaseClient: userkey = config_get('test', 'userkey') def testUserpass(self, vo): + from rucio.client.baseclient import BaseClient """ CLIENTS (BASECLIENT): authenticate with userpass.""" creds = {'username': 'ddmlab', 'password': 'secret'} - client = BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo) - print(client) + BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo) def testUserpassWrongCreds(self, vo): + from rucio.client.baseclient import BaseClient """ CLIENTS (BASECLIENT): try to authenticate with wrong username.""" creds = {'username': 'wrong', 'password': 'secret'} with pytest.raises(CannotAuthenticate): BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo) def testUserpassNoCACert(self, vo): + from rucio.client.baseclient import BaseClient """ CLIENTS (BASECLIENT): authenticate with userpass without ca cert.""" creds = {'username': 'wrong', 'password': 'secret'} with pytest.raises(CannotAuthenticate): BaseClient(account='root', auth_type='userpass', creds=creds, vo=vo) def testx509(self, vo): + from rucio.client.baseclient import BaseClient """ CLIENTS (BASECLIENT): authenticate with x509.""" creds = {'client_cert': self.usercert, 'client_key': self.userkey} - BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo) + logger = setup_logger(verbose=True) + BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo, logger=logger) def testx509NonExistingCert(self, vo): + from rucio.client.baseclient import BaseClient """ CLIENTS (BASECLIENT): authenticate with x509 with missing certificate.""" creds = {'client_cert': '/opt/rucio/etc/web/notthere.crt'} with pytest.raises(CannotAuthenticate): BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo) def testClientProtocolNotSupported(self, vo): + from rucio.client.baseclient import BaseClient """ CLIENTS (BASECLIENT): try to pass an host with a not supported protocol.""" creds = {'username': 'ddmlab', 'password': 'secret'} with pytest.raises(ClientProtocolNotSupported): BaseClient(rucio_host='localhost', auth_host='junk://localhost', account='root', auth_type='userpass', creds=creds, vo=vo) def testRetryOn502AlwaysFail(self, vo): + from rucio.client.baseclient import BaseClient """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes, but fails on repeated errors""" class AlwaysFailWith502(MockServer.Handler): @@ -97,6 +121,8 @@ def do_GET(self): def testRetryOn502SucceedsEventually(self, vo): """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes""" + + from rucio.client.baseclient import BaseClient invocations = [] class FailTwiceWith502(MockServer.Handler): @@ -118,37 +144,186 @@ def do_GET(self, invocations=invocations): assert datetime.utcnow() - start_time > timedelta(seconds=0.9) -class TestRucioClients: - """ To test Clients""" +# Run the whole suite without the config file in place +@pytest.mark.noparallel() +@run_without_config +def test_import_no_config_file(): - cacert = config_get('test', 'cacert') - marker = '$> ' + exitcode, _, err = execute("python -c 'from rucio.client import Client'") - def test_ping(self, vo): - """ PING (CLIENT): Ping Rucio """ - creds = {'username': 'ddmlab', 'password': 'secret'} + assert Config().configfile.split('/') != "rucio.cfg" # Cannot find the config file - client = Client(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo) + assert exitcode == 0 + assert "RuntimeError: Could not load Rucio configuration file." not in err - print(client.ping()) - @pytest.mark.noparallel(reason='We temporarily remove the config file.') - def test_import_without_config_file(self, vo): - """ - The Client should be importable without a config file, since it is - configurable afterwards. +@pytest.mark.noparallel() +@run_without_config +def test_ping_no_config(): + from rucio.client.client import Client + creds = {'username': 'ddmlab', 'password': 'secret'} + ca_cert = '/etc/grid-security/certificates/5fca1cb1.0' + log = setup_logger(verbose=True) + client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log) + client.ping() - We are in a fully configured environment with a default config file. We - thus have to disable the access to it (move it) and make sure to run the - code in a different environment. - """ - configfile = Config().configfile - rename(configfile, f"{configfile}.tmp") - try: - exitcode, _, err = execute("python -c 'from rucio.client import Client'") - print(exitcode, err) - assert exitcode == 0 - assert "RuntimeError: Could not load Rucio configuration file." not in err - finally: - # This is utterly important to not mess up the environment. - rename(f"{configfile}.tmp", configfile) + +@pytest.mark.noparallel() +@run_without_config +def test_account_no_config(): + from rucio.client.client import Client + creds = {'username': 'ddmlab', 'password': 'secret'} + ca_cert = '/etc/grid-security/certificates/5fca1cb1.0' + log = setup_logger(verbose=True) + client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log) + client.list_accounts() + + # List who am i + client.whoami() + name = ''.join(choice(ascii_lowercase) for _ in range(6)) + # Make an account + account_name, type_, email = f"mock_name_{name}", "user", "mock_email" + assert client.add_account(account_name, type_, email) + + # Delete that same account + assert client.delete_account(account_name) + + +@pytest.mark.noparallel() +@run_without_config +def test_did_no_config(did_factory): + from rucio.client.client import Client + + did1, did2 = did_factory.random_dataset_did(), did_factory.random_dataset_did() + + creds = {'username': 'ddmlab', 'password': 'secret'} + ca_cert = '/etc/grid-security/certificates/5fca1cb1.0' + log = setup_logger(verbose=True) + client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log) + + assert client.add_did(scope='mock', name=did1['name'], did_type='dataset') + assert client.add_did(scope='mock', name=did2['name'], did_type='container') + + # Make an attachment + assert client.attach_dids(scope="mock", name=did2['name'], dids=[did1]) + # List did + client.list_dids(scope='mock', filters=[]) + + +@pytest.mark.noparallel() +@run_without_config +@skip_non_atlas # Avoid failure from did format +def test_upload_download_no_config(file_factory, rse_factory): + import os + # Make item + from rucio.client.client import Client + from rucio.client.uploadclient import UploadClient + from rucio.client.downloadclient import DownloadClient + + creds = {'username': 'ddmlab', 'password': 'secret'} + ca_cert = '/etc/grid-security/certificates/5fca1cb1.0' + log = setup_logger(verbose=True) + client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log) + + scope = 'mock' + rse, _ = rse_factory.make_posix_rse() + local_file = file_factory.file_generator() + download_dir = file_factory.base_dir + fn = os.path.basename(local_file) + + # upload a file + status = UploadClient(client).upload([{ + 'path': local_file, + 'rse': rse, + 'did_scope': scope, + 'did_name': fn, + }]) + assert status == 0 + + # download the file + did = f"{scope}:{fn}" + DownloadClient(client).download_dids([{'did': did, 'base_dir': download_dir}]) + + downloaded_file = f"{download_dir}/{scope}/{fn}" + assert os.path.exists(downloaded_file) + + +@pytest.mark.noparallel() +@run_without_config +@skip_non_atlas +def test_replica_no_config(rse_factory, did_factory, file_factory): + from rucio.client.client import Client + from rucio.client.uploadclient import UploadClient + + creds = {'username': 'ddmlab', 'password': 'secret'} + ca_cert = '/etc/grid-security/certificates/5fca1cb1.0' + log = setup_logger(verbose=True) + client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log) + + scope = 'mock' + rse, _ = rse_factory.make_posix_rse() + + def make_replica(): + local_file = file_factory.file_generator() + fn = os.path.basename(local_file) + + # upload a file + UploadClient(client).upload([{ + 'path': local_file, + 'rse': rse, + 'did_scope': scope, + 'did_name': fn, + }]) + + mock_scope = 'mock' + replicas = [{'scope': mock_scope, 'name': fn}] + + return replicas + replica = make_replica() + + assert client.add_replicas(rse, replica) + assert client.list_dataset_replicas("mock", replica[0]['name']) + new_replica = make_replica() + assert client.add_replicas(rse, new_replica) + new_replica[0]['state'] = 'D' + assert client.update_replicas_states(rse, new_replica) + + # TODO Check for an account with the auth to delete files + + +@pytest.mark.noparallel() +@run_without_config +@skip_non_atlas +def test_request_no_config(rse_factory, did_factory, file_factory): + from rucio.client.client import Client + from rucio.client.uploadclient import UploadClient + creds = {'username': 'ddmlab', 'password': 'secret'} + ca_cert = '/etc/grid-security/certificates/5fca1cb1.0' + log = setup_logger(verbose=True) + client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log) + + mock_scope = 'mock' + rse, _ = rse_factory.make_posix_rse() + rse2, _ = rse_factory.make_posix_rse() + + local_file = file_factory.file_generator() + fn = os.path.basename(local_file) + + # upload a file + UploadClient(client).upload([{ + 'path': local_file, + 'rse': rse, + 'did_scope': mock_scope, + 'did_name': fn, + }]) + + did = {"name": fn, "scope": mock_scope} + + client.list_requests(rse, rse2, request_states='Q') + client.list_requests_history(rse, rse2, 'Q') + + with pytest.raises(RequestNotFound): + client.list_request_by_did(did['name'], rse, mock_scope) + + with pytest.raises(RequestNotFound): + client.list_request_history_by_did(did['name'], rse, mock_scope)