From b6fa001cdb49a4db6c469bb9f80b9baee7b91ae4 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sun, 3 Dec 2023 11:01:05 -0500 Subject: [PATCH 01/29] cephadm: add generic methods for sharing namespaces across containers In the future, some sidecar containers will need to share namespaces with the primary container (or each other). Make it easy to set this up by creating a enable_shared_namespaces function and Namespace enum. Signed-off-by: John Mulligan --- src/cephadm/cephadmlib/container_types.py | 63 ++++++++++++++++++++++- src/cephadm/tests/test_util_funcs.py | 62 ++++++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py index 01fbb41d2392d..665c4d89652a6 100644 --- a/src/cephadm/cephadmlib/container_types.py +++ b/src/cephadm/cephadmlib/container_types.py @@ -1,9 +1,11 @@ # container_types.py - container instance wrapper types import copy +import enum +import functools import os -from typing import Dict, List, Optional, Any, Union, Tuple, cast +from typing import Dict, List, Optional, Any, Union, Tuple, Iterable, cast from .call_wrappers import call, call_throws, CallVerbosity from .constants import DEFAULT_TIMEOUT @@ -599,3 +601,62 @@ def extract_uid_gid( raise Error(f'Failed to extract uid/gid for path {ex[0]}: {ex[1]}') raise RuntimeError('uid/gid not found') + + +@functools.lru_cache() +def _opt_key(value: str) -> str: + """Return a (long) option stripped of its value.""" + return value.split('=', 1)[0] + + +def _replace_container_arg(args: List[str], new_arg: str) -> None: + """Remove and replace arguments that have the same `--xyz` part as + the given `new_arg`. If new_arg is expected to have a value it + must be part of the new_arg string following an equal sign (`=`). + The existing arg may be a single or two strings in the input list. + """ + key = _opt_key(new_arg) + has_value = key != new_arg + try: + idx = [_opt_key(v) for v in args].index(key) + if '=' in args[idx] or not has_value: + del args[idx] + else: + del args[idx] + del args[idx] + except ValueError: + pass + args.append(new_arg) + + +class Namespace(enum.Enum): + """General container namespace control options.""" + + cgroupns = 'cgroupns' + cgroup = 'cgroupns' # alias + ipc = 'ipc' + network = 'network' + pid = 'pid' + userns = 'userns' + user = 'userns' # alias + uts = 'uts' + + def to_option(self, value: str) -> str: + return f'--{self}={value}' + + def __str__(self) -> str: + return self.value + + +def enable_shared_namespaces( + args: List[str], + name: str, + ns: Iterable[Namespace], +) -> None: + """Update the args list to contain options that enable container namespace + sharing where name is the name/id of the target container and ns is a list + or set of namespaces that should be shared. + """ + cc = f'container:{name}' + for n in ns: + _replace_container_arg(args, n.to_option(cc)) diff --git a/src/cephadm/tests/test_util_funcs.py b/src/cephadm/tests/test_util_funcs.py index 6b5380711f336..aa64d54b07344 100644 --- a/src/cephadm/tests/test_util_funcs.py +++ b/src/cephadm/tests/test_util_funcs.py @@ -906,3 +906,65 @@ def test_daemon_sub_id_systemd_names(): ) with pytest.raises(ValueError): dsi.service_name + + +@pytest.mark.parametrize( + "args,new_arg,expected", + [ + (['--foo=77'], '--bar', ['--foo=77', '--bar']), + (['--foo=77'], '--foo=12', ['--foo=12']), + ( + ['--foo=77', '--quux=later', '--range=2-5'], + '--quux=now', + ['--foo=77', '--range=2-5', '--quux=now'], + ), + ( + ['--foo=77', '--quux', 'later', '--range=2-5'], + '--quux=now', + ['--foo=77', '--range=2-5', '--quux=now'], + ), + ( + ['--foo=77', '--quux', 'later', '--range=2-5'], + '--jiffy', + ['--foo=77', '--quux', 'later', '--range=2-5', '--jiffy'], + ), + ( + ['--foo=77', '--quux=buff', '--range=2-5'], + '--quux', + ['--foo=77', '--range=2-5', '--quux'], + ), + ], +) +def test_replace_container_args(args, new_arg, expected): + from cephadmlib.container_types import _replace_container_arg + + _args = list(args) # preserve the input so test input is not mutated + _replace_container_arg(_args, new_arg) + assert _args == expected + + + +def test_enable_shared_namespaces(): + from cephadmlib.container_types import enable_shared_namespaces, Namespace + + args = [] + enable_shared_namespaces(args, 'c001d00d', {Namespace.ipc}) + assert args == ['--ipc=container:c001d00d'] + + enable_shared_namespaces( + args, 'c001d00d', [Namespace.uts, Namespace.network] + ) + assert args == [ + '--ipc=container:c001d00d', + '--uts=container:c001d00d', + '--network=container:c001d00d', + ] + + enable_shared_namespaces( + args, 'badd33d5', [Namespace.network] + ) + assert args == [ + '--ipc=container:c001d00d', + '--uts=container:c001d00d', + '--network=container:badd33d5', + ] From d373edf0d6126ab6672f115e439d5bc14f983336 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Tue, 21 Nov 2023 17:11:37 -0500 Subject: [PATCH 02/29] cephadm: add a default constant value for samba server container image Signed-off-by: John Mulligan --- src/cephadm/cephadmlib/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py index 6ca5d39ed9de8..e5c7de8cc43d7 100644 --- a/src/cephadm/cephadmlib/constants.py +++ b/src/cephadm/cephadmlib/constants.py @@ -18,6 +18,7 @@ DEFAULT_JAEGER_COLLECTOR_IMAGE = 'quay.io/jaegertracing/jaeger-collector:1.29' DEFAULT_JAEGER_AGENT_IMAGE = 'quay.io/jaegertracing/jaeger-agent:1.29' DEFAULT_JAEGER_QUERY_IMAGE = 'quay.io/jaegertracing/jaeger-query:1.29' +DEFAULT_SMB_IMAGE = 'quay.io/samba.org/samba-server:devbuilds-centos-amd64' DEFAULT_REGISTRY = 'docker.io' # normalize unqualified digests to this # ------------------------------------------------------------------------------ From 0169fd945e7a1d5ea9dc3969e257cd0ecce0c1c6 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 6 Dec 2023 15:14:31 -0500 Subject: [PATCH 03/29] cephadm: add an SMB daemon module and classes Add an incomplete but largely viable SMB/Samba container daemon form implementation to cephadm. Currently unused but it lays out some of the basics needed to create smb sharing using samba containers under cephadm orchestration. Signed-off-by: John Mulligan --- src/cephadm/cephadm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py index cb86af7df4064..6257fb11d1316 100755 --- a/src/cephadm/cephadm.py +++ b/src/cephadm/cephadm.py @@ -173,6 +173,7 @@ Keepalived, Monitoring, NFSGanesha, + SMB, SNMPGateway, Tracing, NodeProxy, @@ -227,6 +228,7 @@ def get_supported_daemons(): supported_daemons.append(SNMPGateway.daemon_type) supported_daemons.extend(Tracing.components) supported_daemons.append(NodeProxy.daemon_type) + supported_daemons.append(SMB.daemon_type) assert len(supported_daemons) == len(set(supported_daemons)) return supported_daemons From f86e7106a48bd64a36c67377c780c6cf1521bdb5 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 6 Dec 2023 15:14:32 -0500 Subject: [PATCH 04/29] cephadm: import and enable deployment of SMB daemon class Enable the use of the SMB container daemon form class by importing, and thus registering, it. Note that the only way to invoke this feature is by hand rolling some JSON to feed to the `ceph _orch deploy` command. Connecting this with the cephadm mgr module is left as a future task. Signed-off-by: John Mulligan --- src/cephadm/cephadmlib/daemons/__init__.py | 2 + src/cephadm/cephadmlib/daemons/smb.py | 442 +++++++++++++++++++++ 2 files changed, 444 insertions(+) create mode 100644 src/cephadm/cephadmlib/daemons/smb.py diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py index 29f1506948323..1a9d2d568bcfc 100644 --- a/src/cephadm/cephadmlib/daemons/__init__.py +++ b/src/cephadm/cephadmlib/daemons/__init__.py @@ -5,6 +5,7 @@ from .monitoring import Monitoring from .nfs import NFSGanesha from .nvmeof import CephNvmeof +from .smb import SMB from .snmp import SNMPGateway from .tracing import Tracing from .node_proxy import NodeProxy @@ -20,6 +21,7 @@ 'Monitoring', 'NFSGanesha', 'OSD', + 'SMB', 'SNMPGateway', 'Tracing', 'NodeProxy', diff --git a/src/cephadm/cephadmlib/daemons/smb.py b/src/cephadm/cephadmlib/daemons/smb.py new file mode 100644 index 0000000000000..00103ac8e4b4e --- /dev/null +++ b/src/cephadm/cephadmlib/daemons/smb.py @@ -0,0 +1,442 @@ +import enum +import json +import pathlib +import logging + +from typing import List, Dict, Tuple, Optional, Any + +from .. import context_getters +from .. import daemon_form +from .. import data_utils +from .. import deployment_utils +from .. import file_utils +from ..constants import DEFAULT_SMB_IMAGE +from ..container_daemon_form import ContainerDaemonForm, daemon_to_container +from ..container_engines import Podman +from ..container_types import ( + CephContainer, + InitContainer, + Namespace, + SidecarContainer, + enable_shared_namespaces, +) +from ..context import CephadmContext +from ..daemon_identity import DaemonIdentity, DaemonSubIdentity +from ..deploy import DeploymentType +from ..exceptions import Error +from ..net_utils import EndPoint + + +logger = logging.getLogger() + + +class Features(enum.Enum): + DOMAIN = 'domain' + CLUSTERED = 'clustered' + + @classmethod + def valid(cls, value: str) -> bool: + # workaround for older python versions + try: + cls(value) + return True + except ValueError: + return False + + +class Config: + instance_id: str + source_config: str + samba_debug_level: int + debug_delay: int + domain_member: bool + clustered: bool + join_sources: List[str] + custom_dns: List[str] + smb_port: int + ceph_config_entity: str + + def __init__( + self, + *, + instance_id: str, + source_config: str, + domain_member: bool, + clustered: bool, + samba_debug_level: int = 0, + debug_delay: int = 0, + join_sources: Optional[List[str]] = None, + custom_dns: Optional[List[str]] = None, + smb_port: int = 0, + ceph_config_entity: str = 'client.admin', + ) -> None: + self.instance_id = instance_id + self.source_config = source_config + self.domain_member = domain_member + self.clustered = clustered + self.samba_debug_level = samba_debug_level + self.debug_delay = debug_delay + self.join_sources = join_sources or [] + self.custom_dns = custom_dns or [] + self.smb_port = smb_port + self.ceph_config_entity = ceph_config_entity + + def __str__(self) -> str: + return ( + f'SMB Config[id={self.instance_id},' + f' source_config={self.source_config},' + f' domain_member={self.domain_member},' + f' clustered={self.clustered}]' + ) + + +class SambaContainerCommon: + def __init__( + self, + cfg: Config, + ) -> None: + self.cfg = cfg + + def name(self) -> str: + raise NotImplementedError('samba container name') + + def envs(self) -> Dict[str, str]: + cfg_uris = [self.cfg.source_config] + environ = { + 'SAMBA_CONTAINER_ID': self.cfg.instance_id, + 'SAMBACC_CONFIG': json.dumps(cfg_uris), + } + if self.cfg.ceph_config_entity: + environ['SAMBACC_CEPH_ID'] = f'name={self.cfg.ceph_config_entity}' + return environ + + def envs_list(self) -> List[str]: + return [f'{k}={v}' for (k, v) in self.envs().items()] + + def args(self) -> List[str]: + args = [] + if self.cfg.samba_debug_level: + args.append(f'--samba-debug-level={self.cfg.samba_debug_level}') + if self.cfg.debug_delay: + args.append(f'--debug-delay={self.cfg.debug_delay}') + return args + + def container_args(self) -> List[str]: + return [] + + +class SMBDContainer(SambaContainerCommon): + def name(self) -> str: + return 'smbd' + + def args(self) -> List[str]: + return super().args() + ['run', 'smbd'] + + def container_args(self) -> List[str]: + cargs = [] + if self.cfg.smb_port: + cargs.append(f'--publish={self.cfg.smb_port}:{self.cfg.smb_port}') + return cargs + + +class WinbindContainer(SambaContainerCommon): + def name(self) -> str: + return 'winbindd' + + def args(self) -> List[str]: + return super().args() + ['run', 'winbindd'] + + +class ConfigInitContainer(SambaContainerCommon): + def name(self) -> str: + return 'config' + + def args(self) -> List[str]: + return super().args() + ['init'] + + +class MustJoinContainer(SambaContainerCommon): + def name(self) -> str: + return 'mustjoin' + + def args(self) -> List[str]: + args = super().args() + ['must-join'] + for join_src in self.cfg.join_sources: + args.append(f'-j{join_src}') + return args + + def container_args(self) -> List[str]: + cargs = [] + for dns in self.cfg.custom_dns: + cargs.append(f'--dns={dns}') + return cargs + + +class ConfigWatchContainer(SambaContainerCommon): + def name(self) -> str: + return 'configwatch' + + def args(self) -> List[str]: + return super().args() + ['update-config', '--watch'] + + +class ContainerLayout: + init_containers: List[SambaContainerCommon] + primary: SambaContainerCommon + supplemental: List[SambaContainerCommon] + + def __init__( + self, + init_containers: List[SambaContainerCommon], + primary: SambaContainerCommon, + supplemental: List[SambaContainerCommon], + ) -> None: + self.init_containers = init_containers + self.primary = primary + self.supplemental = supplemental + + +@daemon_form.register +class SMB(ContainerDaemonForm): + """Provides a form for SMB containers.""" + + daemon_type = 'smb' + default_image = DEFAULT_SMB_IMAGE + + @classmethod + def for_daemon_type(cls, daemon_type: str) -> bool: + return cls.daemon_type == daemon_type + + def __init__(self, ctx: CephadmContext, ident: DaemonIdentity): + assert ident.daemon_type == self.daemon_type + self._identity = ident + self._instance_cfg: Optional[Config] = None + self._files: Dict[str, str] = {} + self._raw_configs: Dict[str, Any] = context_getters.fetch_configs(ctx) + self._config_keyring = context_getters.get_config_and_keyring(ctx) + self._cached_layout: Optional[ContainerLayout] = None + self.smb_port = 445 + logger.debug('Created SMB ContainerDaemonForm instance') + + def validate(self) -> None: + if self._instance_cfg is not None: + return + + configs = self._raw_configs + instance_id = configs.get('cluster_id', '') + source_config = configs.get('config_uri', '') + join_sources = configs.get('join_sources', []) + custom_dns = configs.get('custom_dns', []) + instance_features = configs.get('features', []) + files = data_utils.dict_get(configs, 'files', {}) + ceph_config_entity = configs.get('config_auth_entity', '') + + if not instance_id: + raise Error('invalid instance (cluster) id') + if not source_config: + raise Error('invalid configuration source uri') + invalid_features = { + f for f in instance_features if not Features.valid(f) + } + if invalid_features: + raise Error( + f'invalid instance features: {", ".join(invalid_features)}' + ) + if Features.CLUSTERED.value in instance_features: + raise NotImplementedError('clustered instance') + + self._instance_cfg = Config( + instance_id=instance_id, + source_config=source_config, + join_sources=join_sources, + custom_dns=custom_dns, + domain_member=Features.DOMAIN.value in instance_features, + clustered=Features.CLUSTERED.value in instance_features, + samba_debug_level=6, + smb_port=self.smb_port, + ceph_config_entity=ceph_config_entity, + ) + self._files = files + logger.debug('SMB Instance Config: %s', self._instance_cfg) + logger.debug('Configured files: %s', self._files) + + @property + def _cfg(self) -> Config: + self.validate() + assert self._instance_cfg + return self._instance_cfg + + @property + def instance_id(self) -> str: + return self._cfg.instance_id + + @property + def source_config(self) -> str: + return self._cfg.source_config + + @classmethod + def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'SMB': + return cls(ctx, ident) + + @property + def identity(self) -> DaemonIdentity: + return self._identity + + def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]: + return 0, 0 + + def config_and_keyring( + self, ctx: CephadmContext + ) -> Tuple[Optional[str], Optional[str]]: + return self._config_keyring + + def _layout(self) -> ContainerLayout: + if self._cached_layout: + return self._cached_layout + init_ctrs: List[SambaContainerCommon] = [] + ctrs: List[SambaContainerCommon] = [] + + init_ctrs.append(ConfigInitContainer(self._cfg)) + ctrs.append(ConfigWatchContainer(self._cfg)) + + if self._cfg.domain_member: + init_ctrs.append(MustJoinContainer(self._cfg)) + ctrs.append(WinbindContainer(self._cfg)) + + smbd = SMBDContainer(self._cfg) + self._cached_layout = ContainerLayout(init_ctrs, smbd, ctrs) + return self._cached_layout + + def _to_init_container( + self, ctx: CephadmContext, smb_ctr: SambaContainerCommon + ) -> InitContainer: + volume_mounts: Dict[str, str] = {} + container_args: List[str] = smb_ctr.container_args() + self.customize_container_mounts(ctx, volume_mounts) + # XXX: is this needed? if so, can this be simplified + if isinstance(ctx.container_engine, Podman): + ctx.container_engine.update_mounts(ctx, volume_mounts) + identity = DaemonSubIdentity.from_parent( + self.identity, smb_ctr.name() + ) + return InitContainer( + ctx, + entrypoint='', + image=ctx.image or self.default_image, + identity=identity, + args=smb_ctr.args(), + container_args=container_args, + envs=smb_ctr.envs_list(), + volume_mounts=volume_mounts, + ) + + def _to_sidecar_container( + self, ctx: CephadmContext, smb_ctr: SambaContainerCommon + ) -> SidecarContainer: + volume_mounts: Dict[str, str] = {} + container_args: List[str] = smb_ctr.container_args() + self.customize_container_mounts(ctx, volume_mounts) + shared_ns = { + Namespace.ipc, + Namespace.network, + Namespace.pid, + } + if isinstance(ctx.container_engine, Podman): + # XXX: is this needed? if so, can this be simplified + ctx.container_engine.update_mounts(ctx, volume_mounts) + # docker doesn't support sharing the uts namespace with other + # containers. It may not be entirely needed on podman but it gives + # me warm fuzzies to make sure it gets shared. + shared_ns.add(Namespace.uts) + enable_shared_namespaces( + container_args, self.identity.container_name, shared_ns + ) + identity = DaemonSubIdentity.from_parent( + self.identity, smb_ctr.name() + ) + return SidecarContainer( + ctx, + entrypoint='', + image=ctx.image or self.default_image, + identity=identity, + container_args=container_args, + args=smb_ctr.args(), + envs=smb_ctr.envs_list(), + volume_mounts=volume_mounts, + init=False, + remove=True, + ) + + def container(self, ctx: CephadmContext) -> CephContainer: + ctr = daemon_to_container(ctx, self, host_network=False) + # We want to share the IPC ns between the samba containers for one + # instance. Cephadm's default, host ipc, is not what we want. + # Unsetting it works fine for podman but docker (on ubuntu 22.04) needs + # to be expliclty told that ipc of the primary container must be + # shareable. + ctr.ipc = 'shareable' + return deployment_utils.to_deployment_container(ctx, ctr) + + def init_containers(self, ctx: CephadmContext) -> List[InitContainer]: + return [ + self._to_init_container(ctx, smb_ctr) + for smb_ctr in self._layout().init_containers + ] + + def sidecar_containers( + self, ctx: CephadmContext + ) -> List[SidecarContainer]: + return [ + self._to_sidecar_container(ctx, smb_ctr) + for smb_ctr in self._layout().supplemental + ] + + def customize_container_envs( + self, ctx: CephadmContext, envs: List[str] + ) -> None: + clayout = self._layout() + envs.extend(clayout.primary.envs_list()) + + def customize_process_args( + self, ctx: CephadmContext, args: List[str] + ) -> None: + clayout = self._layout() + args.extend(clayout.primary.args()) + + def customize_container_args( + self, ctx: CephadmContext, args: List[str] + ) -> None: + args.extend(self._layout().primary.container_args()) + + def customize_container_mounts( + self, + ctx: CephadmContext, + mounts: Dict[str, str], + ) -> None: + self.validate() + data_dir = pathlib.Path(self.identity.data_dir(ctx.data_dir)) + etc_samba_ctr = str(data_dir / 'etc-samba-container') + lib_samba = str(data_dir / 'lib-samba') + run_samba = str(data_dir / 'run') + config = str(data_dir / 'config') + keyring = str(data_dir / 'keyring') + mounts[etc_samba_ctr] = '/etc/samba/container:z' + mounts[lib_samba] = '/var/lib/samba:z' + mounts[run_samba] = '/run:z' # TODO: make this a shared tmpfs + mounts[config] = '/etc/ceph/ceph.conf:z' + mounts[keyring] = '/etc/ceph/keyring:z' + + def customize_container_endpoints( + self, endpoints: List[EndPoint], deployment_type: DeploymentType + ) -> None: + if not any(ep.port == self.smb_port for ep in endpoints): + endpoints.append(EndPoint('0.0.0.0', self.smb_port)) + + def prepare_data_dir(self, data_dir: str, uid: int, gid: int) -> None: + self.validate() + ddir = pathlib.Path(data_dir) + file_utils.makedirs(ddir / 'etc-samba-container', uid, gid, 0o770) + file_utils.makedirs(ddir / 'lib-samba', uid, gid, 0o770) + file_utils.makedirs(ddir / 'run', uid, gid, 0o770) + if self._files: + file_utils.populate_files(data_dir, self._files, uid, gid) From 3b0f33188abef97d8e6ff774b03947ba23e180d0 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 6 Dec 2023 15:14:32 -0500 Subject: [PATCH 05/29] cephadm: add a basic deployment test for an smb daemon Signed-off-by: John Mulligan --- src/cephadm/tests/test_deploy.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py index 26d5c56b7563a..0be62ad021eb2 100644 --- a/src/cephadm/tests/test_deploy.py +++ b/src/cephadm/tests/test_deploy.py @@ -534,3 +534,33 @@ def test_deploy_and_rm_iscsi(cephadm_fs, funkypatch): assert not drop_in.exists() assert not drop_in.parent.exists() assert not tcmu_sidecar.exists() + + +def test_deploy_smb_container(cephadm_fs, funkypatch): + mocks = _common_patches(funkypatch) + fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7' + with with_cephadm_ctx([]) as ctx: + ctx.container_engine = mock_podman() + ctx.fsid = fsid + ctx.name = 'smb.b01s' + ctx.image = 'quay.io/essembee/samba-server:latest' + ctx.reconfig = False + ctx.config_blobs = { + 'cluster_id': 'smb1', + 'config_uri': 'http://localhost:9876/smb.json', + 'config': 'SAMPLE', + 'keyring': 'SOMETHING', + } + _cephadm._common_deploy(ctx) + + basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/smb.b01s') + assert basedir.is_dir() + with open(basedir / 'unit.run') as f: + runfile_lines = f.read().splitlines() + assert 'podman' in runfile_lines[-1] + assert runfile_lines[-1].endswith('quay.io/essembee/samba-server:latest --samba-debug-level=6 run smbd') + assert f'-v {basedir}/etc-samba-container:/etc/samba/container:z' in runfile_lines[-1] + assert f'-v {basedir}/lib-samba:/var/lib/samba:z' in runfile_lines[-1] + assert '-e SAMBA_CONTAINER_ID=smb1' in runfile_lines[-1] + assert '-e \'SAMBACC_CONFIG=["http://localhost:9876/smb.json"]\'' in runfile_lines[-1] + assert '--publish' in runfile_lines[-1] From f8160ed11b046bc747bbf045173fb222688ed669 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Tue, 16 Jan 2024 15:37:27 -0500 Subject: [PATCH 06/29] cephadm: fix issue joining to ad by using a virtual hostname The not-a-real-fqdn hostname that the containers got were causing performance issues joining AD (and running testjoin and winbind). Define a virtual hostname that can be passed in from the service or automatically derived from the system's hostname. Signed-off-by: John Mulligan --- src/cephadm/cephadmlib/daemons/smb.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/cephadm/cephadmlib/daemons/smb.py b/src/cephadm/cephadmlib/daemons/smb.py index 00103ac8e4b4e..bd92b97e7f196 100644 --- a/src/cephadm/cephadmlib/daemons/smb.py +++ b/src/cephadm/cephadmlib/daemons/smb.py @@ -1,7 +1,8 @@ import enum import json -import pathlib import logging +import pathlib +import socket from typing import List, Dict, Tuple, Optional, Any @@ -55,6 +56,7 @@ class Config: custom_dns: List[str] smb_port: int ceph_config_entity: str + vhostname: str def __init__( self, @@ -69,6 +71,7 @@ def __init__( custom_dns: Optional[List[str]] = None, smb_port: int = 0, ceph_config_entity: str = 'client.admin', + vhostname: str = '', ) -> None: self.instance_id = instance_id self.source_config = source_config @@ -80,6 +83,7 @@ def __init__( self.custom_dns = custom_dns or [] self.smb_port = smb_port self.ceph_config_entity = ceph_config_entity + self.vhostname = vhostname def __str__(self) -> str: return ( @@ -90,6 +94,15 @@ def __str__(self) -> str: ) +def _container_dns_args(cfg: Config) -> List[str]: + cargs = [] + for dns in cfg.custom_dns: + cargs.append(f'--dns={dns}') + if cfg.vhostname: + cargs.append(f'--hostname={cfg.vhostname}') + return cargs + + class SambaContainerCommon: def __init__( self, @@ -136,6 +149,7 @@ def container_args(self) -> List[str]: cargs = [] if self.cfg.smb_port: cargs.append(f'--publish={self.cfg.smb_port}:{self.cfg.smb_port}') + cargs.extend(_container_dns_args(self.cfg)) return cargs @@ -166,9 +180,7 @@ def args(self) -> List[str]: return args def container_args(self) -> List[str]: - cargs = [] - for dns in self.cfg.custom_dns: - cargs.append(f'--dns={dns}') + cargs = _container_dns_args(self.cfg) return cargs @@ -230,6 +242,7 @@ def validate(self) -> None: instance_features = configs.get('features', []) files = data_utils.dict_get(configs, 'files', {}) ceph_config_entity = configs.get('config_auth_entity', '') + vhostname = configs.get('virtual_hostname', '') if not instance_id: raise Error('invalid instance (cluster) id') @@ -244,6 +257,11 @@ def validate(self) -> None: ) if Features.CLUSTERED.value in instance_features: raise NotImplementedError('clustered instance') + if not vhostname: + # if a virtual hostname is not provided, generate one by prefixing + # the cluster/instanced id to the system hostname + hname = socket.getfqdn() + vhostname = f'{instance_id}-{hname}' self._instance_cfg = Config( instance_id=instance_id, @@ -255,6 +273,7 @@ def validate(self) -> None: samba_debug_level=6, smb_port=self.smb_port, ceph_config_entity=ceph_config_entity, + vhostname=vhostname, ) self._files = files logger.debug('SMB Instance Config: %s', self._instance_cfg) From 07b44900e8c8bef434e848bf7aa84d18d78d8bd6 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 14:33:20 -0500 Subject: [PATCH 07/29] mgr/cephadm: fix test failure on newer python Tests that touch this enum fail for me locally but pass in the CI. This seems to be due to new enum related behavior in Python 3.11. See: https://blog.pecar.me/python-enum Instead of fixing it as suggested in the above blog, adding a __str__ method works on all python versions I care to know about. Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pybind/mgr/cephadm/utils.py b/src/pybind/mgr/cephadm/utils.py index 3aedfbd86f008..1ba3e48454a20 100644 --- a/src/pybind/mgr/cephadm/utils.py +++ b/src/pybind/mgr/cephadm/utils.py @@ -57,6 +57,9 @@ class SpecialHostLabels(str, Enum): def to_json(self) -> str: return self.value + def __str__(self) -> str: + return self.value + def name_to_config_section(name: str) -> ConfEntity: """ From 96456aaf46d13ee29529c4fb031a90f0e4a795a8 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 14:37:17 -0500 Subject: [PATCH 08/29] mgr/orchestrator: clean up import style In the seemingly never-ending fight against line continuations and just blatting tons of stuff onto single lines another small victory is won. Signed-off-by: John Mulligan --- src/pybind/mgr/orchestrator/module.py | 33 +++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 96a2d91040124..3e80621ef8386 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -26,13 +26,32 @@ from mgr_module import MgrModule, HandleCommandResult, Option from object_format import Format -from ._interface import OrchestratorClientMixin, DeviceLightLoc, _cli_read_command, \ - raise_if_exception, _cli_write_command, OrchestratorError, \ - NoOrchestrator, OrchestratorValidationError, NFSServiceSpec, \ - RGWSpec, InventoryFilter, InventoryHost, HostSpec, CLICommandMeta, \ - ServiceDescription, DaemonDescription, IscsiServiceSpec, json_to_generic_spec, \ - GenericSpec, DaemonDescriptionStatus, SNMPGatewaySpec, MDSSpec, TunedProfileSpec, \ - NvmeofServiceSpec +from ._interface import ( + CLICommandMeta, + DaemonDescription, + DaemonDescriptionStatus, + DeviceLightLoc, + GenericSpec, + HostSpec, + InventoryFilter, + InventoryHost, + IscsiServiceSpec, + MDSSpec, + NFSServiceSpec, + NoOrchestrator, + NvmeofServiceSpec, + OrchestratorClientMixin, + OrchestratorError, + OrchestratorValidationError, + RGWSpec, + SNMPGatewaySpec, + ServiceDescription, + TunedProfileSpec, + _cli_read_command, + _cli_write_command, + json_to_generic_spec, + raise_if_exception, +) def nice_delta(now: datetime.datetime, t: Optional[datetime.datetime], suffix: str = '') -> str: From 35028e15789dc3600143a434301625d094b24475 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 16:05:27 -0500 Subject: [PATCH 09/29] mgr/orchestrator: fix the sorting of the imports While ceph doesn't enforce sorted imports I prefer them when possible. I had once sorted these imports but then nvmeof came along an ruined things. Put nvmeof back in it's place. Signed-off-by: John Mulligan --- src/pybind/mgr/orchestrator/_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index f0fb2c429069c..7e1b57466e3b9 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -38,11 +38,11 @@ class Protocol: # type: ignore IscsiServiceSpec, MDSSpec, NFSServiceSpec, + NvmeofServiceSpec, RGWSpec, SNMPGatewaySpec, ServiceSpec, TunedProfileSpec, - NvmeofServiceSpec ) from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.hostspec import HostSpec, SpecValidationError From a500f42d1a6faf5dc2607dd3ec425c5af7585128 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 15:49:12 -0500 Subject: [PATCH 10/29] mgr/cephadm: reformat the _service_classes variable Reformat the _service_classes variable so that it uses a multi-line list with a single item on each line in a more black-ish style that is more readable (especially if you use code-folding wisely). Sort the list while we're at it. Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/module.py | 33 ++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 697fad0f05adf..d30f2e5b5b6c2 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -661,12 +661,33 @@ def __init__(self, *args: Any, **kwargs: Any): self.migration = Migrations(self) _service_classes: Sequence[Type[CephadmService]] = [ - OSDService, NFSService, MonService, MgrService, MdsService, - RgwService, RbdMirrorService, GrafanaService, AlertmanagerService, - PrometheusService, NodeExporterService, LokiService, PromtailService, CrashService, IscsiService, - IngressService, CustomContainerService, CephfsMirrorService, NvmeofService, - CephadmAgent, CephExporterService, SNMPGatewayService, ElasticSearchService, - JaegerQueryService, JaegerAgentService, JaegerCollectorService, NodeProxy + AlertmanagerService, + CephExporterService, + CephadmAgent, + CephfsMirrorService, + CrashService, + CustomContainerService, + ElasticSearchService, + GrafanaService, + IngressService, + IscsiService, + JaegerAgentService, + JaegerCollectorService, + JaegerQueryService, + LokiService, + MdsService, + MgrService, + MonService, + NFSService, + NodeExporterService, + NodeProxy, + NvmeofService, + OSDService, + PrometheusService, + PromtailService, + RbdMirrorService, + RgwService, + SNMPGatewayService, ] # https://github.com/python/mypy/issues/8993 From 41e2b27817c783f3b4b142441ed827e1827482d6 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 5 Jan 2024 10:24:10 -0500 Subject: [PATCH 11/29] mgr/cephadm: refactor keyring simplification out of get_keyring_with_caps Refactor get_keyring_with_caps such that the keyring simplification code is moved into a new function that can be used in other locations. get_keyring_with_caps will now call the new function to return the simplified & consistent keyring output. Signed-off-by: John Mulligan --- .../mgr/cephadm/services/cephadmservice.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index d211bbaa309f4..6e3ee927341cd 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -55,6 +55,24 @@ def get_auth_entity(daemon_type: str, daemon_id: str, host: str = "") -> AuthEnt raise OrchestratorError(f"unknown daemon type {daemon_type}") +def simplified_keyring(entity: str, contents: str) -> str: + # strip down keyring + # - don't include caps (auth get includes them; get-or-create does not) + # - use pending key if present + key = None + for line in contents.splitlines(): + if ' = ' not in line: + continue + line = line.strip() + (ls, rs) = line.split(' = ', 1) + if ls == 'key' and not key: + key = rs + if ls == 'pending key': + key = rs + keyring = f'[{entity}]\nkey = {key}\n' + return keyring + + class CephadmDaemonDeploySpec: # typing.NamedTuple + Generic is broken in py36 def __init__(self, host: str, daemon_id: str, @@ -307,22 +325,7 @@ def get_keyring_with_caps(self, entity: AuthEntity, caps: List[str]) -> str: }) if err: raise OrchestratorError(f"Unable to fetch keyring for {entity}: {err}") - - # strip down keyring - # - don't include caps (auth get includes them; get-or-create does not) - # - use pending key if present - key = None - for line in keyring.splitlines(): - if ' = ' not in line: - continue - line = line.strip() - (ls, rs) = line.split(' = ', 1) - if ls == 'key' and not key: - key = rs - if ls == 'pending key': - key = rs - keyring = f'[{entity}]\nkey = {key}\n' - return keyring + return simplified_keyring(entity, keyring) def _inventory_get_fqdn(self, hostname: str) -> str: """Get a host's FQDN with its hostname. From 4fc2697fb1e4dc71b480db275aa4e54c2b66d018 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 19:20:45 -0500 Subject: [PATCH 12/29] python-common: reformat ServiceSpec class level service type lists Reformat the ServiceSpec classes properties KNOWN_SERVICE_TYPES and REQUIRES_SERVICE_ID. These were previously strings that were converted to lists via a call to split. With a string there's very little a human or a tool can do to validate the content. Changing these into proper lists in the source code brings clarity of intent and the ability to analyze the code. Because there's no semantic difference what services are listed where (this means the type could probably be a set - a quest for another day) I also took the opportunity to sort the contents of the lists and add some basic comments for what these lists are for. It also removes the use of (ugly, IMO) line continuations. The downside is that it makes more total lines, but if that bugs you - use code folding :-). Signed-off-by: John Mulligan --- .../ceph/deployment/service_spec.py | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 24f5c646461b7..c1fb11c60d785 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -753,12 +753,52 @@ class ServiceSpec(object): This structure is supposed to be enough information to start the services. """ - KNOWN_SERVICE_TYPES = 'alertmanager crash grafana iscsi nvmeof loki promtail mds mgr mon nfs ' \ - 'node-exporter osd prometheus rbd-mirror rgw agent ceph-exporter ' \ - 'container ingress cephfs-mirror snmp-gateway jaeger-tracing ' \ - 'elasticsearch jaeger-agent jaeger-collector jaeger-query ' \ - 'node-proxy'.split() - REQUIRES_SERVICE_ID = 'iscsi nvmeof mds nfs rgw container ingress '.split() + + # list of all service type names that a ServiceSpec can be cast info + KNOWN_SERVICE_TYPES = [ + 'agent', + 'alertmanager', + 'ceph-exporter', + 'cephfs-mirror', + 'container', + 'crash', + 'elasticsearch', + 'grafana', + 'ingress', + 'iscsi', + 'jaeger-agent', + 'jaeger-collector', + 'jaeger-query', + 'jaeger-tracing', + 'loki', + 'mds', + 'mgr', + 'mon', + 'nfs', + 'node-exporter', + 'node-proxy', + 'nvmeof', + 'osd', + 'prometheus', + 'promtail', + 'rbd-mirror', + 'rgw', + 'snmp-gateway', + ] + + # list of all service type names that require/get assigned a service_id value. + # if a service is not listed here it *will not* be assigned a service_id even + # if it is present in the JSON/YAML input + REQUIRES_SERVICE_ID = [ + 'container', + 'ingress', + 'iscsi', + 'mds', + 'nfs', + 'nvmeof', + 'rgw', + ] + MANAGED_CONFIG_OPTIONS = [ 'mds_join_fs', ] From 4f655c5e1894812ad983111276f188b4fd61aebe Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 16:10:37 -0500 Subject: [PATCH 13/29] python-common: define a new SMBSpec service spec type Signed-off-by: John Mulligan --- .../ceph/deployment/service_spec.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index c1fb11c60d785..5d2e410e58c9b 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -783,6 +783,7 @@ class ServiceSpec(object): 'promtail', 'rbd-mirror', 'rgw', + 'smb', 'snmp-gateway', ] @@ -797,6 +798,7 @@ class ServiceSpec(object): 'nfs', 'nvmeof', 'rgw', + 'smb', ] MANAGED_CONFIG_OPTIONS = [ @@ -830,6 +832,7 @@ def _cls(cls: Type[ServiceSpecT], service_type: str) -> Type[ServiceSpecT]: 'jaeger-collector': TracingSpec, 'jaeger-query': TracingSpec, 'jaeger-tracing': TracingSpec, + SMBSpec.service_type: SMBSpec, }.get(service_type, cls) if ret == ServiceSpec and not service_type: raise SpecValidationError('Spec needs a "service_type" key.') @@ -2372,3 +2375,85 @@ def validate(self) -> None: yaml.add_representer(CephExporterSpec, ServiceSpec.yaml_representer) + + +class SMBSpec(ServiceSpec): + service_type = 'smb' + _valid_features = {'domain'} + + def __init__( + self, + # --- common service spec args --- + service_type: str = 'smb', + service_id: Optional[str] = None, + placement: Optional[PlacementSpec] = None, + count: Optional[int] = None, + config: Optional[Dict[str, str]] = None, + unmanaged: bool = False, + preview_only: bool = False, + networks: Optional[List[str]] = None, + # --- smb specific values --- + # cluster_id - a name identifying the smb "cluster" this daemon + # is part of. A cluster may be made up of one or more services + # sharing a common configuration. + cluster_id: str = '', + # features - a list of terms enabling specific deployment features. + # terms include: 'domain' to enable Active Dir. Domain membership. + features: Optional[List[str]] = None, + # config_uri - a pseudo-uri that resolves to a configuration source + # that the samba-container can load. A ceph based samba container will + # be typically storing configuration in rados (rados:// prefix) + config_uri: str = '', + # join_sources - a list of pseudo-uris that resolve to a (JSON) blob + # containing data the samba-container can use to join a domain. A ceph + # based samba container may typically use a rados uri or a mon + # config-key store uri (example: + # `rados:mon-config-key:smb/config/mycluster/join1.json`). + join_sources: Optional[List[str]] = None, + # custom_dns - a list of IP addresses that will be set up as custom + # dns servers for the samba container. + custom_dns: Optional[List[str]] = None, + # include_ceph_users - A list of ceph auth entity names that will be + # automatically added to the ceph keyring provided to the samba + # container. + include_ceph_users: Optional[List[str]] = None, + # --- genearal tweaks --- + extra_container_args: Optional[GeneralArgList] = None, + extra_entrypoint_args: Optional[GeneralArgList] = None, + custom_configs: Optional[List[CustomConfig]] = None, + ) -> None: + if service_type != self.service_type: + raise ValueError(f'invalid service_type: {service_type!r}') + super().__init__( + self.service_type, + service_id=service_id, + placement=placement, + count=count, + config=config, + unmanaged=unmanaged, + preview_only=preview_only, + networks=networks, + extra_container_args=extra_container_args, + extra_entrypoint_args=extra_entrypoint_args, + custom_configs=custom_configs, + ) + self.cluster_id = cluster_id + self.features = features or [] + self.config_uri = config_uri + self.join_sources = join_sources or [] + self.custom_dns = custom_dns or [] + self.include_ceph_users = include_ceph_users or [] + self.validate() + + def validate(self) -> None: + if not self.cluster_id: + raise ValueError('a valid cluster_id is required') + if not self.config_uri: + raise ValueError('a valid config_uri is required') + if self.features: + invalid = set(self.features).difference(self._valid_features) + if invalid: + raise ValueError(f'invalid feature flags: {", ".join(invalid)}') + + +yaml.add_representer(SMBSpec, ServiceSpec.yaml_representer) From a88cf505a051298996b6be99d6d55a91a7684467 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 16:10:11 -0500 Subject: [PATCH 14/29] mgr/cephadm: add a new smb ceph service subclass Will be used in a later commit to implement deploying smb instances. Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/services/smb.py | 123 +++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 src/pybind/mgr/cephadm/services/smb.py diff --git a/src/pybind/mgr/cephadm/services/smb.py b/src/pybind/mgr/cephadm/services/smb.py new file mode 100644 index 0000000000000..920c4ef02f741 --- /dev/null +++ b/src/pybind/mgr/cephadm/services/smb.py @@ -0,0 +1,123 @@ +import logging +from typing import Any, Dict, Iterator, List, Tuple, cast + +from ceph.deployment.service_spec import ServiceSpec, SMBSpec + +from orchestrator import DaemonDescription +from .cephadmservice import ( + AuthEntity, + CephService, + CephadmDaemonDeploySpec, + simplified_keyring, +) + +logger = logging.getLogger(__name__) + + +class SMBService(CephService): + TYPE = 'smb' + + def config(self, spec: ServiceSpec) -> None: + assert self.TYPE == spec.service_type + logger.warning('config is a no-op') + + def prepare_create( + self, daemon_spec: CephadmDaemonDeploySpec + ) -> CephadmDaemonDeploySpec: + assert self.TYPE == daemon_spec.daemon_type + logger.debug('smb prepare_create') + daemon_spec.final_config, daemon_spec.deps = self.generate_config( + daemon_spec + ) + return daemon_spec + + def generate_config( + self, daemon_spec: CephadmDaemonDeploySpec + ) -> Tuple[Dict[str, Any], List[str]]: + logger.debug('smb generate_config') + assert self.TYPE == daemon_spec.daemon_type + smb_spec = cast( + SMBSpec, self.mgr.spec_store[daemon_spec.service_name].spec + ) + config_blobs: Dict[str, Any] = {} + + config_blobs['cluster_id'] = smb_spec.cluster_id + config_blobs['features'] = smb_spec.features + config_blobs['config_uri'] = smb_spec.config_uri + if smb_spec.join_sources: + config_blobs['join_sources'] = smb_spec.join_sources + if smb_spec.custom_dns: + config_blobs['custom_dns'] = smb_spec.custom_dns + ceph_users = smb_spec.include_ceph_users or [] + config_blobs.update( + self._ceph_config_and_keyring_for( + smb_spec, daemon_spec.daemon_id, ceph_users + ) + ) + logger.debug('smb generate_config: %r', config_blobs) + return config_blobs, [] + + def config_dashboard( + self, daemon_descrs: List[DaemonDescription] + ) -> None: + # TODO ??? + logger.warning('config_dashboard is a no-op') + + def get_auth_entity(self, daemon_id: str, host: str = "") -> AuthEntity: + # We want a clear, distinct auth entity for fetching the config versus + # data path access. + return AuthEntity(f'client.{self.TYPE}.config.{daemon_id}') + + def _rados_uri_to_pool(self, uri: str) -> str: + """Given a psudo-uri possibly pointing to an object in a pool, return + the name of the pool if a rados uri, otherwise return an empty string. + """ + if not uri.startswith('rados://'): + return '' + pool = uri[8:].lstrip('/').split('/')[0] + logger.debug('extracted pool %r from uri %r', pool, uri) + return pool + + def _allow_config_key_command(self, name: str) -> str: + # permit the samba container config access to the mon config key store + # with keys like smb/config//*. + return f'allow command "config-key get" with "key" prefix "smb/config/{name}/"' + + def _pools_in_spec(self, smb_spec: SMBSpec) -> Iterator[str]: + uris = [smb_spec.config_uri] + uris.extend(smb_spec.join_sources or []) + for uri in uris: + pool = self._rados_uri_to_pool(uri) + if pool: + yield pool + + def _key_for_user(self, entity: str) -> str: + ret, keyring, err = self.mgr.mon_command({ + 'prefix': 'auth get', + 'entity': entity, + }) + if ret != 0: + raise ValueError(f'no auth key for user: {entity!r}') + return '\n' + simplified_keyring(entity, keyring) + + def _ceph_config_and_keyring_for( + self, smb_spec: SMBSpec, daemon_id: str, ceph_users: List[str] + ) -> Dict[str, str]: + ackc = self._allow_config_key_command(smb_spec.cluster_id) + wanted_caps = ['mon', f'allow r, {ackc}'] + pools = list(self._pools_in_spec(smb_spec)) + if pools: + wanted_caps.append('osd') + wanted_caps.append( + ', '.join(f'allow r pool={pool}' for pool in pools) + ) + entity = self.get_auth_entity(daemon_id) + keyring = self.get_keyring_with_caps(entity, wanted_caps) + # add additional data-path users to the ceph keyring + for ceph_user in ceph_users: + keyring += self._key_for_user(ceph_user) + return { + 'config': self.mgr.get_minimal_ceph_conf(), + 'keyring': keyring, + 'config_auth_entity': entity, + } From c5e4912fd5fd43e21525952ca1d295dac10e2bbe Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Thu, 4 Jan 2024 16:38:08 -0500 Subject: [PATCH 15/29] mgr/cepahdm: add various touch points to enable smb service Add the smb service by name or by type to one of the many, many touch points in the orchestrator and cephadm packages needed to get the orchestrator aware of smb. Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/module.py | 9 +++++- src/pybind/mgr/cephadm/utils.py | 4 +-- src/pybind/mgr/orchestrator/_interface.py | 12 ++++++-- src/pybind/mgr/orchestrator/module.py | 37 +++++++++++++++++++++++ 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index d30f2e5b5b6c2..8e81375917222 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -76,6 +76,7 @@ NodeExporterService, SNMPGatewayService, LokiService, PromtailService from .services.jaeger import ElasticSearchService, JaegerAgentService, JaegerCollectorService, JaegerQueryService from .services.node_proxy import NodeProxy +from .services.smb import SMBService from .schedule import HostAssignment from .inventory import Inventory, SpecStore, HostCache, AgentCache, EventStore, \ ClientKeyringStore, ClientKeyringSpec, TunedProfileStore, NodeProxyCache @@ -687,6 +688,7 @@ def __init__(self, *args: Any, **kwargs: Any): PromtailService, RbdMirrorService, RgwService, + SMBService, SNMPGatewayService, ] @@ -3275,7 +3277,8 @@ def _apply_service_spec(self, spec: ServiceSpec) -> str: 'elasticsearch': PlacementSpec(count=1), 'jaeger-agent': PlacementSpec(host_pattern='*'), 'jaeger-collector': PlacementSpec(count=1), - 'jaeger-query': PlacementSpec(count=1) + 'jaeger-query': PlacementSpec(count=1), + SMBService.TYPE: PlacementSpec(count=1), } spec.placement = defaults[spec.service_type] elif spec.service_type in ['mon', 'mgr'] and \ @@ -3405,6 +3408,10 @@ def apply_container(self, spec: ServiceSpec) -> str: def apply_snmp_gateway(self, spec: ServiceSpec) -> str: return self._apply(spec) + @handle_orch_error + def apply_smb(self, spec: ServiceSpec) -> str: + return self._apply(spec) + @handle_orch_error def set_unmanaged(self, service_name: str, value: bool) -> str: return self.spec_store.set_unmanaged(service_name, value) diff --git a/src/pybind/mgr/cephadm/utils.py b/src/pybind/mgr/cephadm/utils.py index 1ba3e48454a20..3673fbf621cb9 100644 --- a/src/pybind/mgr/cephadm/utils.py +++ b/src/pybind/mgr/cephadm/utils.py @@ -36,7 +36,7 @@ class CephadmNoImage(Enum): # these daemons do not use the ceph image. There are other daemons # that also don't use the ceph image, but we only care about those # that are part of the upgrade order here -NON_CEPH_IMAGE_TYPES = MONITORING_STACK_TYPES + ['nvmeof'] +NON_CEPH_IMAGE_TYPES = MONITORING_STACK_TYPES + ['nvmeof', 'smb'] # Used for _run_cephadm used for check-host etc that don't require an --image parameter cephadmNoImage = CephadmNoImage.token @@ -66,7 +66,7 @@ def name_to_config_section(name: str) -> ConfEntity: Map from daemon names to ceph entity names (as seen in config) """ daemon_type = name.split('.', 1)[0] - if daemon_type in ['rgw', 'rbd-mirror', 'nfs', 'crash', 'iscsi', 'ceph-exporter', 'nvmeof']: + if daemon_type in ['rgw', 'rbd-mirror', 'nfs', 'crash', 'iscsi', 'ceph-exporter', 'nvmeof', 'smb']: return ConfEntity('client.' + name) elif daemon_type in ['mon', 'osd', 'mds', 'mgr', 'client']: return ConfEntity(name) diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index 7e1b57466e3b9..f5ddbe6515b24 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -40,6 +40,7 @@ class Protocol: # type: ignore NFSServiceSpec, NvmeofServiceSpec, RGWSpec, + SMBSpec, SNMPGatewaySpec, ServiceSpec, TunedProfileSpec, @@ -582,6 +583,7 @@ def apply(self, specs: Sequence["GenericSpec"], no_overwrite: bool = False) -> L 'ingress': self.apply_ingress, 'snmp-gateway': self.apply_snmp_gateway, 'host': self.add_host, + 'smb': self.apply_smb, } def merge(l: OrchResult[List[str]], r: OrchResult[str]) -> OrchResult[List[str]]: # noqa: E741 @@ -819,6 +821,10 @@ def apply_snmp_gateway(self, spec: SNMPGatewaySpec) -> OrchResult[str]: """Update an existing snmp gateway service""" raise NotImplementedError() + def apply_smb(self, spec: SMBSpec) -> OrchResult[str]: + """Update a smb gateway service""" + raise NotImplementedError() + def apply_tuned_profiles(self, specs: List[TunedProfileSpec], no_overwrite: bool) -> OrchResult[str]: """Add or update an existing tuned profile""" raise NotImplementedError() @@ -917,7 +923,8 @@ def daemon_type_to_service(dtype: str) -> str: 'elasticsearch': 'elasticsearch', 'jaeger-agent': 'jaeger-agent', 'jaeger-collector': 'jaeger-collector', - 'jaeger-query': 'jaeger-query' + 'jaeger-query': 'jaeger-query', + 'smb': 'smb', } return mapping[dtype] @@ -951,7 +958,8 @@ def service_to_daemon_types(stype: str) -> List[str]: 'jaeger-agent': ['jaeger-agent'], 'jaeger-collector': ['jaeger-collector'], 'jaeger-query': ['jaeger-query'], - 'jaeger-tracing': ['elasticsearch', 'jaeger-query', 'jaeger-collector', 'jaeger-agent'] + 'jaeger-tracing': ['elasticsearch', 'jaeger-query', 'jaeger-collector', 'jaeger-agent'], + 'smb': ['smb'], } return mapping[stype] diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 3e80621ef8386..d1ba6ea867552 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -44,6 +44,7 @@ OrchestratorError, OrchestratorValidationError, RGWSpec, + SMBSpec, SNMPGatewaySpec, ServiceDescription, TunedProfileSpec, @@ -1820,6 +1821,42 @@ def _apply_jaeger(self, specs: List[ServiceSpec] = spec.get_tracing_specs() return self._apply_misc(specs, dry_run, format, no_overwrite) + @_cli_write_command('orch apply smb') + def _apply_smb( + self, + cluster_id: str, + config_uri: str, + features: str = '', + join_sources: Optional[List[str]] = None, + custom_dns: Optional[List[str]] = None, + include_ceph_users: Optional[List[str]] = None, + placement: Optional[str] = None, + unmanaged: bool = False, + dry_run: bool = False, + format: Format = Format.plain, + no_overwrite: bool = False, + ) -> HandleCommandResult: + """Apply an SMB network file system gateway service configuration.""" + + _features = features.replace(',', ' ').split() + spec = SMBSpec( + service_id=cluster_id, + placement=PlacementSpec.from_string(placement), + unmanaged=unmanaged, + preview_only=dry_run, + cluster_id=cluster_id, + features=_features, + config_uri=config_uri, + join_sources=join_sources, + custom_dns=custom_dns, + include_ceph_users=include_ceph_users, + ) + + spec.validate() # force any validation exceptions to be caught correctly + # The previous comment makes no sense to JJM. But when in rome. + + return self._apply_misc([spec], dry_run, format, no_overwrite) + @_cli_write_command('orch set-unmanaged') def _set_unmanaged(self, service_name: str) -> HandleCommandResult: """Set 'unmanaged: true' for the given service name""" From 3985325e6983e3440ebb73a5b328071c096ac027 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 15 Dec 2023 13:15:19 -0500 Subject: [PATCH 16/29] mgr/cephadm: add the samba container image for smb daemons Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/module.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 8e81375917222..8771c9a1e8c03 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -132,6 +132,7 @@ def os_exit_noop(status: int) -> None: DEFAULT_JAEGER_COLLECTOR_IMAGE = 'quay.io/jaegertracing/jaeger-collector:1.29' DEFAULT_JAEGER_AGENT_IMAGE = 'quay.io/jaegertracing/jaeger-agent:1.29' DEFAULT_JAEGER_QUERY_IMAGE = 'quay.io/jaegertracing/jaeger-query:1.29' +DEFAULT_SAMBA_IMAGE = 'quay.io/samba.org/samba-server:devbuilds-centos-amd64' # ------------------------------------------------------------------------------ @@ -288,6 +289,11 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, default=DEFAULT_JAEGER_QUERY_IMAGE, desc='Jaeger query container image', ), + Option( + 'container_image_samba', + default=DEFAULT_SAMBA_IMAGE, + desc='Samba/SMB container image', + ), Option( 'warn_on_stray_hosts', type='bool', @@ -552,6 +558,7 @@ def __init__(self, *args: Any, **kwargs: Any): self.container_image_jaeger_agent = '' self.container_image_jaeger_collector = '' self.container_image_jaeger_query = '' + self.container_image_samba = '' self.warn_on_stray_hosts = True self.warn_on_stray_daemons = True self.warn_on_failed_host_check = True @@ -1617,6 +1624,8 @@ def _get_container_image(self, daemon_name: str) -> Optional[str]: image = None elif daemon_type == 'snmp-gateway': image = self.container_image_snmp_gateway + elif daemon_type == SMBService.TYPE: + image = self.container_image_samba else: assert False, daemon_type From 0847ee2ee4531f2bb02ee3cc2a290c4c10fe4330 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 5 Jan 2024 10:45:08 -0500 Subject: [PATCH 17/29] mgr/cephadm: simplify _get_container_image a bit Because the "if-ladder" was only ever assigning a single variable with a value it can be directly replaced by a dict & dict-lookup which is much more succinct. Also take the opportunity to sort the (non-comment) lines as there's no meaning to the previous order and this makes it easier for a reader to scan through. Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/module.py | 61 +++++++++++++------------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 8771c9a1e8c03..b93cb7f3096fa 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1591,43 +1591,32 @@ def _get_container_image(self, daemon_name: str) -> Optional[str]: utils.name_to_config_section(daemon_name), 'container_image' )).strip() - elif daemon_type == 'prometheus': - image = self.container_image_prometheus - elif daemon_type == 'nvmeof': - image = self.container_image_nvmeof - elif daemon_type == 'grafana': - image = self.container_image_grafana - elif daemon_type == 'alertmanager': - image = self.container_image_alertmanager - elif daemon_type == 'node-exporter': - image = self.container_image_node_exporter - elif daemon_type == 'loki': - image = self.container_image_loki - elif daemon_type == 'promtail': - image = self.container_image_promtail - elif daemon_type == 'haproxy': - image = self.container_image_haproxy - elif daemon_type == 'keepalived': - image = self.container_image_keepalived - elif daemon_type == 'elasticsearch': - image = self.container_image_elasticsearch - elif daemon_type == 'jaeger-agent': - image = self.container_image_jaeger_agent - elif daemon_type == 'jaeger-collector': - image = self.container_image_jaeger_collector - elif daemon_type == 'jaeger-query': - image = self.container_image_jaeger_query - elif daemon_type == CustomContainerService.TYPE: - # The image can't be resolved, the necessary information - # is only available when a container is deployed (given - # via spec). - image = None - elif daemon_type == 'snmp-gateway': - image = self.container_image_snmp_gateway - elif daemon_type == SMBService.TYPE: - image = self.container_image_samba else: - assert False, daemon_type + images = { + 'alertmanager': self.container_image_alertmanager, + 'elasticsearch': self.container_image_elasticsearch, + 'grafana': self.container_image_grafana, + 'haproxy': self.container_image_haproxy, + 'jaeger-agent': self.container_image_jaeger_agent, + 'jaeger-collector': self.container_image_jaeger_collector, + 'jaeger-query': self.container_image_jaeger_query, + 'keepalived': self.container_image_keepalived, + 'loki': self.container_image_loki, + 'node-exporter': self.container_image_node_exporter, + 'nvmeof': self.container_image_nvmeof, + 'prometheus': self.container_image_prometheus, + 'promtail': self.container_image_promtail, + 'snmp-gateway': self.container_image_snmp_gateway, + # The image can't be resolved here, the necessary information + # is only available when a container is deployed (given + # via spec). + CustomContainerService.TYPE: None, + SMBService.TYPE: self.container_image_samba, + } + try: + image = images[daemon_type] + except KeyError: + raise ValueError(f'no image for {daemon_type}') self.log.debug('%s container image %s' % (daemon_name, image)) From 9a58843dde69d2016be019d3e501d70022720e11 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 13 Dec 2023 19:36:46 -0500 Subject: [PATCH 18/29] mgr/cephadm: add some tests for the new smb service Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/tests/test_services.py | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py index cb7f618d37b4d..48123b8ea1747 100644 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ b/src/pybind/mgr/cephadm/tests/test_services.py @@ -17,6 +17,7 @@ from cephadm.services.osd import OSDService from cephadm.services.monitoring import GrafanaService, AlertmanagerService, PrometheusService, \ NodeExporterService, LokiService, PromtailService +from cephadm.services.smb import SMBSpec from cephadm.module import CephadmOrchestrator from ceph.deployment.service_spec import ( AlertManagerSpec, @@ -2984,3 +2985,122 @@ def test_deploy_custom_container_with_init_ctrs( [], stdin=json.dumps(expected), ) + + +class TestSMB: + @patch("cephadm.module.CephadmOrchestrator.get_unique_name") + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_deploy_smb( + self, _run_cephadm, _get_uname, cephadm_module: CephadmOrchestrator + ): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + _get_uname.return_value = 'tango.briskly' + + spec = SMBSpec( + cluster_id='foxtrot', + config_uri='rados://.smb/foxtrot/config.json', + ) + + expected = { + 'fsid': 'fsid', + 'name': 'smb.tango.briskly', + 'image': '', + 'deploy_arguments': [], + 'params': {}, + 'meta': { + 'service_name': 'smb', + 'ports': [], + 'ip': None, + 'deployed_by': [], + 'rank': None, + 'rank_generation': None, + 'extra_container_args': None, + 'extra_entrypoint_args': None, + }, + 'config_blobs': { + 'cluster_id': 'foxtrot', + 'features': [], + 'config_uri': 'rados://.smb/foxtrot/config.json', + 'config': '', + 'keyring': '[client.smb.config.tango.briskly]\nkey = None\n', + 'config_auth_entity': 'client.smb.config.tango.briskly', + }, + } + with with_host(cephadm_module, 'hostx'): + with with_service(cephadm_module, spec): + _run_cephadm.assert_called_with( + 'hostx', + 'smb.tango.briskly', + ['_orch', 'deploy'], + [], + stdin=json.dumps(expected), + ) + + @patch("cephadm.module.CephadmOrchestrator.get_unique_name") + @patch("cephadm.serve.CephadmServe._run_cephadm") + def test_deploy_smb_join_dns( + self, _run_cephadm, _get_uname, cephadm_module: CephadmOrchestrator + ): + _run_cephadm.side_effect = async_side_effect(('{}', '', 0)) + _get_uname.return_value = 'tango.briskly' + + spec = SMBSpec( + cluster_id='foxtrot', + features=['domain'], + config_uri='rados://.smb/foxtrot/config2.json', + join_sources=[ + 'rados://.smb/foxtrot/join1.json', + 'rados:mon-config-key:smb/config/foxtrot/join2.json', + ], + custom_dns=['10.8.88.103'], + include_ceph_users=[ + 'client.smb.fs.cephfs.share1', + 'client.smb.fs.cephfs.share2', + 'client.smb.fs.fs2.share3', + ], + ) + + expected = { + 'fsid': 'fsid', + 'name': 'smb.tango.briskly', + 'image': '', + 'deploy_arguments': [], + 'params': {}, + 'meta': { + 'service_name': 'smb', + 'ports': [], + 'ip': None, + 'deployed_by': [], + 'rank': None, + 'rank_generation': None, + 'extra_container_args': None, + 'extra_entrypoint_args': None, + }, + 'config_blobs': { + 'cluster_id': 'foxtrot', + 'features': ['domain'], + 'config_uri': 'rados://.smb/foxtrot/config2.json', + 'join_sources': [ + 'rados://.smb/foxtrot/join1.json', + 'rados:mon-config-key:smb/config/foxtrot/join2.json', + ], + 'custom_dns': ['10.8.88.103'], + 'config': '', + 'keyring': ( + '[client.smb.config.tango.briskly]\nkey = None\n\n' + '[client.smb.fs.cephfs.share1]\nkey = None\n\n' + '[client.smb.fs.cephfs.share2]\nkey = None\n\n' + '[client.smb.fs.fs2.share3]\nkey = None\n' + ), + 'config_auth_entity': 'client.smb.config.tango.briskly', + }, + } + with with_host(cephadm_module, 'hostx'): + with with_service(cephadm_module, spec): + _run_cephadm.assert_called_with( + 'hostx', + 'smb.tango.briskly', + ['_orch', 'deploy'], + [], + stdin=json.dumps(expected), + ) From 4e897de3225aa8624c3e0188236f85f523088ea2 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 10 Feb 2024 09:06:53 -0500 Subject: [PATCH 19/29] doc/cephadm: add a file documenting the smb service Signed-off-by: John Mulligan --- doc/cephadm/services/index.rst | 1 + doc/cephadm/services/smb.rst | 205 +++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 doc/cephadm/services/smb.rst diff --git a/doc/cephadm/services/index.rst b/doc/cephadm/services/index.rst index c1da5d15f8955..df8b3b421699a 100644 --- a/doc/cephadm/services/index.rst +++ b/doc/cephadm/services/index.rst @@ -19,6 +19,7 @@ for details on individual services: monitoring snmp-gateway tracing + smb Service Status ============== diff --git a/doc/cephadm/services/smb.rst b/doc/cephadm/services/smb.rst new file mode 100644 index 0000000000000..8ca70c5baf23d --- /dev/null +++ b/doc/cephadm/services/smb.rst @@ -0,0 +1,205 @@ +.. _deploy-cephadm-smb-samba: + +=========== +SMB Service +=========== + +.. note:: Only the SMB3 protocol is supported. + +.. warning:: + + SMB support is under active development and many features may be + missing or immature. Additionally, a Manager module to automate + SMB clusters and SMB shares is in development. Once that feature + is developed it will be the preferred method for managing + SMB on ceph. + + +Deploying Samba Containers +========================== + +Cephadm deploys `Samba `_ servers using container images +built by the `samba-container project `_. + +In order to host SMB Shares with access to CephFS file systems, deploy +Samba Containers with the following command: + +.. prompt:: bash # + + orch apply smb [--features ...] [--placement ...] ... + +There are a number of additional parameters that the command accepts. See +the Service Specification for a description of these options. + +Service Specification +===================== + +An SMB Service can be applied using a specification. An example in YAML follows: + +.. code-block:: yaml + + service_type: smb + service_id: tango + placement: + hosts: + - ceph0 + spec: + cluster_id: tango + features: + - domain + config_uri: rados://.smb/tango/scc.toml + custom_dns: + - "192.168.76.204" + join_sources: + - "rados:mon-config-key:smb/config/tango/join1.json" + include_ceph_users: + - client.smb.fs.cluster.tango + +The specification can then be applied by running the following command: + +.. prompt:: bash # + + ceph orch apply -i smb.yaml + + +Service Spec Options +-------------------- + +Fields specific to the ``spec`` section of the SMB Service are described below. + +cluster_id + A short name identifying the SMB "cluster". In this case a cluster is + simply a management unit of one or more Samba services sharing a common + configuration, and may not provide actual clustering or availability + mechanisms. + +features + A list of pre-defined terms enabling specific deployment characteristics. + An empty list is valid. Supported terms: + + * ``domain``: Enable domain member mode + +config_uri + A string containing a (standard or de-facto) URI that identifies a + configuration source that should be loaded by the samba-container as the + primary configuration file. + Supported URI schemes include ``http:``, ``https:``, ``rados:``, and + ``rados:mon-config-key:``. + +join_sources + A list of strings with (standard or de-facto) URI values that will + be used to identify where authentication data that will be used to + perform domain joins are located. Each join source is tried in sequence + until one succeeds. + See ``config_uri`` for the supported list of URI schemes. + +custom_dns + A list of IP addresses that will be used as the DNS servers for a Samba + container. This features allows Samba Containers to integrate with + Active Directory even if the Ceph host nodes are not tied into the Active + Directory DNS domain(s). + +include_ceph_users: + A list of cephx user (aka entity) names that the Samba Containers may use. + The cephx keys for each user in the list will automatically be added to + the keyring in the container. + + +Configuring an SMB Service +-------------------------- + +.. warning:: + + A Manager module for SMB is under active development. Once that module + is available it will be the preferred method for managing Samba on Ceph + in an end-to-end manner. The following discussion is provided for the sake + of completeness and to explain how the software layers interact. + +Creating an SMB Service spec is not sufficient for complete operation of a +Samba Container on Ceph. It is important to create valid configurations and +place them in locations that the container can read. The complete specification +of these configurations is out of scope for this document. You can refer to the +`documentation for Samba `_ as +well as the `samba server container +`_ +and the `configuation file +`_ +it accepts. + +When one has composed a configuration it should be stored in a location +that the Samba Container can access. The recommended approach for running +Samba Containers within Ceph orchestration is to store the configuration +in the Ceph cluster. There are two ways to store the configuration +in ceph: + +RADOS +~~~~~ + +A configuration file can be stored as a RADOS object in a pool +named ``.smb``. Within the pool there should be a namespace named after the +``cluster_id`` value. The URI used to identify this resource should be +constructed like ``rados://.smb//``. Example: +``rados://.smb/tango/config.json``. + +The containers are automatically deployed with cephx keys allowing access to +resources in these pools and namespaces. As long as this scheme is used +no additional configuration to read the object is needed. + +To copy a configuration file to a RADOS pool, use the ``rados`` command line +tool. For example: + +.. prompt:: bash # + + # assuming your config file is /tmp/config.json + rados --pool=.smb --namespace=tango put config.json /tmp/config.json + +MON Key/Value Store +~~~~~~~~~~~~~~~~~~~ + +A configuration file can be stored as a value in the Ceph Monitor Key/Value +store. The key must be named after the cluster like so: +``smb/config//``. This results in a URI that can be used to +identify this configuration constructed like +``rados:mon-config-key:smb/config//``. +Example: ``rados:mon-config-key:smb/config/tango/config.json``. + +The containers are automatically deployed with cephx keys allowing access to +resources with the key-prefix ``smb/config//``. As long as this +scheme is used no additional configuration to read the value is needed. + +To copy a configuration file into the Key/Value store use the ``ceph config-key +put ...`` tool. For example: + +.. prompt:: bash # + + # assuming your config file is /tmp/config.json + ceph config-key set smb/config/tango/config.json -i /tmp/config.json + + +HTTP/HTTPS +~~~~~~~~~~ + +A configuration file can be stored on an HTTP(S) server and automatically read +by the Samba Container. Managing a configuration file on HTTP(S) is left as an +exercise for the reader. + +.. note:: All URI schemes are supported by parameters that accept URIs. Each + scheme has different performance and security characteristics. + + +Limitations +=========== + +A non-exhaustive list of important limitations for the SMB service follows: + +* DNS is a critical component of Active Directory. If one is configuring the + SMB service for domain membership, either the Ceph host node must be + configured so that it can resolve the Active Directory (AD) domain or the + ``custom_dns`` option may be used. In both cases DNS hosts for the AD domain + must still be reachable from whatever network segment the ceph cluster is on. +* Proper clustering/high-availability/"transparent state migration" is not yet + supported. If a placement causes more than service to be created these + services will act independently and may lead to unexpected behavior if clients + access the same files at once. +* Services must bind to TCP port 445. Running multiple SMB services on the same + node is not yet supported and will trigger a port-in-use conflict. From a99dc99589a0b81792abab1d14db6aad3ef9f2b8 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Tue, 20 Feb 2024 18:28:58 -0500 Subject: [PATCH 20/29] qa/tasks: add a new cephadm task for setting up samba ad dc Add a new task function to cephadm.py that sets up a container running the Samba based domain controller on a node using podman or docker. Much of the function actually deals with disabling systemd-resolved because that service conflicts with the DNS server component of the DC. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 250 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 248ce68e12f19..927cf6c4599ff 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -9,6 +9,7 @@ import logging import os import re +import time import uuid import yaml @@ -24,6 +25,7 @@ from teuthology.orchestra import run from teuthology.orchestra.daemon import DaemonGroup from teuthology.config import config as teuth_config +from teuthology.exceptions import ConfigError, CommandFailedError from textwrap import dedent from tasks.cephfs.filesystem import MDSCluster, Filesystem from tasks.util import chacra @@ -1746,6 +1748,254 @@ def initialize_config(ctx, config): yield +def _disable_systemd_resolved(ctx, remote): + r = remote.run(args=['ss', '-lunH'], stdout=StringIO()) + # this heuristic tries to detect if systemd-resolved is running + if '%lo:53' not in r.stdout.getvalue(): + return + log.info('Disabling systemd-resolved on %s', remote.shortname) + # Samba AD DC container DNS support conflicts with resolved stub + # resolver when using host networking. And we want host networking + # because it is the simplest thing to set up. We therefore will turn + # off the stub resolver. + r = remote.run( + args=['sudo', 'cat', '/etc/systemd/resolved.conf'], + stdout=StringIO(), + ) + resolved_conf = r.stdout.getvalue() + setattr(ctx, 'orig_resolved_conf', resolved_conf) + new_resolved_conf = ( + resolved_conf + '\n# EDITED BY TEUTHOLOGY: deploy_samba_ad_dc\n' + ) + if '[Resolve]' not in new_resolved_conf.splitlines(): + new_resolved_conf += '[Resolve]\n' + new_resolved_conf += 'DNSStubListener=no\n' + remote.write_file( + path='/etc/systemd/resolved.conf', + data=new_resolved_conf, + sudo=True, + ) + remote.run(args=['sudo', 'systemctl', 'restart', 'systemd-resolved']) + r = remote.run(args=['ss', '-lunH'], stdout=StringIO()) + assert '%lo:53' not in r.stdout.getvalue() + # because docker is a big fat persistent deamon, we need to bounce it + # after resolved is restarted + remote.run(args=['sudo', 'systemctl', 'restart', 'docker']) + + +def _reset_systemd_resolved(ctx, remote): + orig_resolved_conf = getattr(ctx, 'orig_resolved_conf', None) + if not orig_resolved_conf: + return # no orig_resolved_conf means nothing to reset + log.info('Resetting systemd-resolved state on %s', remote.shortname) + remote.write_file( + path='/etc/systemd/resolved.conf', + data=orig_resolved_conf, + sudo=True, + ) + remote.run(args=['sudo', 'systemctl', 'restart', 'systemd-resolved']) + setattr(ctx, 'orig_resolved_conf', None) + + +def _samba_ad_dc_conf(ctx, remote, cengine): + # this config has not been tested outside of smithi nodes. it's possible + # that this will break when used elsewhere because we have to list + # interfaces explicitly. Later I may add a feature to sambacc to exclude + # known-unwanted interfaces that having to specify known good interfaces. + cf = { + "samba-container-config": "v0", + "configs": { + "demo": { + "instance_features": ["addc"], + "domain_settings": "sink", + "instance_name": "dc1", + } + }, + "domain_settings": { + "sink": { + "realm": "DOMAIN1.SINK.TEST", + "short_domain": "DOMAIN1", + "admin_password": "Passw0rd", + "interfaces": { + "exclude_pattern": "^docker[0-9]+$", + }, + } + }, + "domain_groups": { + "sink": [ + {"name": "supervisors"}, + {"name": "employees"}, + {"name": "characters"}, + {"name": "bulk"}, + ] + }, + "domain_users": { + "sink": [ + { + "name": "bwayne", + "password": "1115Rose.", + "given_name": "Bruce", + "surname": "Wayne", + "member_of": ["supervisors", "characters", "employees"], + }, + { + "name": "ckent", + "password": "1115Rose.", + "given_name": "Clark", + "surname": "Kent", + "member_of": ["characters", "employees"], + }, + { + "name": "user0", + "password": "1115Rose.", + "given_name": "George0", + "surname": "Hue-Sir", + "member_of": ["bulk"], + }, + { + "name": "user1", + "password": "1115Rose.", + "given_name": "George1", + "surname": "Hue-Sir", + "member_of": ["bulk"], + }, + { + "name": "user2", + "password": "1115Rose.", + "given_name": "George2", + "surname": "Hue-Sir", + "member_of": ["bulk"], + }, + { + "name": "user3", + "password": "1115Rose.", + "given_name": "George3", + "surname": "Hue-Sir", + "member_of": ["bulk"], + }, + ] + }, + } + cf_json = json.dumps(cf) + remote.run(args=['sudo', 'mkdir', '-p', '/var/tmp/samba']) + remote.write_file( + path='/var/tmp/samba/container.json', data=cf_json, sudo=True + ) + return [ + '--volume=/var/tmp/samba:/etc/samba-container:ro', + '-eSAMBACC_CONFIG=/etc/samba-container/container.json', + ] + + +@contextlib.contextmanager +def deploy_samba_ad_dc(ctx, config): + role = config.get('role') + ad_dc_image = config.get( + 'ad_dc_image', 'quay.io/samba.org/samba-ad-server:latest' + ) + samba_client_image = config.get( + 'samba_client_image', 'quay.io/samba.org/samba-client:latest' + ) + test_user_pass = config.get('test_user_pass', 'DOMAIN1\\ckent%1115Rose.') + if not role: + raise ConfigError( + "you must specify a role to allocate a host for the AD DC" + ) + (remote,) = ctx.cluster.only(role).remotes.keys() + ip = remote.ssh.get_transport().getpeername()[0] + cengine = 'podman' + try: + log.info("Testing if podman is available") + remote.run(args=['sudo', cengine, '--help']) + except CommandFailedError: + log.info("Failed to find podman. Using docker") + cengine = 'docker' + remote.run(args=['sudo', cengine, 'pull', ad_dc_image]) + remote.run(args=['sudo', cengine, 'pull', samba_client_image]) + _disable_systemd_resolved(ctx, remote) + remote.run( + args=[ + 'sudo', + 'mkdir', + '-p', + '/var/lib/samba/container/logs', + '/var/lib/samba/container/data', + ] + ) + remote.run( + args=[ + 'sudo', + cengine, + 'run', + '-d', + '--name=samba-ad', + '--network=host', + '--privileged', + ] + + _samba_ad_dc_conf(ctx, remote, cengine) + + [ad_dc_image] + ) + + # test that the ad dc is running and basically works + connected = False + samba_client_container_cmd = [ + 'sudo', + cengine, + 'run', + '--rm', + '--net=host', + f'--dns={ip}', + '-eKRB5_CONFIG=/dev/null', + samba_client_image, + ] + for idx in range(10): + time.sleep((2 ** (1 + idx)) / 8) + log.info("Probing SMB status of DC %s, idx=%s", ip, idx) + cmd = samba_client_container_cmd + [ + 'smbclient', + '-U', + test_user_pass, + '//domain1.sink.test/sysvol', + '-c', + 'ls', + ] + try: + remote.run(args=cmd) + connected = True + log.info("SMB status probe succeeded") + break + except CommandFailedError: + pass + if not connected: + raise RuntimeError('failed to connect to AD DC SMB share') + + setattr(ctx, 'samba_ad_dc_ip', ip) + setattr(ctx, 'samba_client_container_cmd', samba_client_container_cmd) + try: + yield + finally: + try: + remote.run(args=['sudo', cengine, 'stop', 'samba-ad']) + except CommandFailedError: + log.error("Failed to stop samba-ad container") + try: + remote.run(args=['sudo', cengine, 'rm', 'samba-ad']) + except CommandFailedError: + log.error("Failed to remove samba-ad container") + remote.run( + args=[ + 'sudo', + 'rm', + '-rf', + '/var/lib/samba/container/logs', + '/var/lib/samba/container/data', + ] + ) + _reset_systemd_resolved(ctx, remote) + setattr(ctx, 'samba_ad_dc_ip', None) + setattr(ctx, 'samba_client_container_cmd', None) + + @contextlib.contextmanager def task(ctx, config): """ From 2a917e23ca6b3d2a4b90a40a07a1b5ae299b3924 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 24 Feb 2024 14:26:36 -0500 Subject: [PATCH 21/29] qa/tasks: allow passing stdin string to cephadm shell commands There are cases where I want to pass some large-ish strings to ceph commands executed via cephadm shell. Allow items within the commands list to be dicts containing a command (as before) and an optional stdin variable. This change also supports possible future extensions as well. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 927cf6c4599ff..daf36fb84cc50 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -1421,10 +1421,17 @@ def shell(ctx, config): (remote,) = ctx.cluster.only(role).remotes.keys() log.info('Running commands on role %s host %s', role, remote.name) if isinstance(cmd, list): - for c in cmd: - _shell(ctx, cluster_name, remote, - ['bash', '-c', c], - extra_cephadm_args=args) + for cobj in cmd: + sh_cmd, stdin = _shell_command(cobj) + _shell( + ctx, + cluster_name, + remote, + ['bash', '-c', sh_cmd], + extra_cephadm_args=args, + stdin=stdin, + ) + else: assert isinstance(cmd, str) _shell(ctx, cluster_name, remote, @@ -1432,6 +1439,16 @@ def shell(ctx, config): extra_cephadm_args=args) +def _shell_command(obj): + if isinstance(obj, str): + return obj, None + if isinstance(obj, dict): + cmd = obj['cmd'] + stdin = obj.get('stdin', None) + return cmd, stdin + raise ValueError(f'invalid command item: {obj!r}') + + def apply(ctx, config): """ Apply spec From 3ec0bfa9eb5daf6b8b0101a50d0323918cdb7d31 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Mon, 26 Feb 2024 13:47:04 -0500 Subject: [PATCH 22/29] qa/tasks: add a cephadm.exclude role Add a cephadm.exclude role that excludes a test node from cluster setup and related commands. I need this as I have test node that will be set up as an AD Domain Controller for testing Samba and do not want that node to be have *any* other services running on it. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 52 ++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index daf36fb84cc50..a7069c19fe7b1 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -112,6 +112,19 @@ def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs): ) +def _cephadm_remotes(ctx, log_excluded=False): + out = [] + for remote, roles in ctx.cluster.remotes.items(): + if any(r.startswith('cephadm.exclude') for r in roles): + if log_excluded: + log.info( + f'Remote {remote.shortname} excluded from cephadm cluster by role' + ) + continue + out.append((remote, roles)) + return out + + def build_initial_config(ctx, config): cluster_name = config['cluster'] @@ -138,7 +151,7 @@ def distribute_iscsi_gateway_cfg(ctx, conf_data): These will help in iscsi clients with finding trusted_ip_list. """ log.info('Distributing iscsi-gateway.cfg...') - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): remote.write_file( path='/etc/ceph/iscsi-gateway.cfg', data=conf_data, @@ -367,13 +380,14 @@ def _fetch_stable_branch_cephadm_from_chacra(ctx, config, cluster_name): def _rm_cluster(ctx, cluster_name): log.info('Removing cluster...') - ctx.cluster.run(args=[ - 'sudo', - ctx.cephadm, - 'rm-cluster', - '--fsid', ctx.ceph[cluster_name].fsid, - '--force', - ]) + for remote, _ in _cephadm_remotes(ctx): + remote.run(args=[ + 'sudo', + ctx.cephadm, + 'rm-cluster', + '--fsid', ctx.ceph[cluster_name].fsid, + '--force', + ]) def _rm_cephadm(ctx): @@ -785,7 +799,7 @@ def ceph_bootstrap(ctx, config): check_status=False) # add other hosts - for remote in ctx.cluster.remotes.keys(): + for remote, roles in _cephadm_remotes(ctx, log_excluded=True): if remote == bootstrap_remote: continue @@ -869,7 +883,7 @@ def ceph_mons(ctx, config): # This is the old way of adding mons that works with the (early) octopus # cephadm scheduler. num_mons = 1 - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): for mon in [r for r in roles if teuthology.is_type('mon', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(mon) @@ -908,7 +922,7 @@ def ceph_mons(ctx, config): break else: nodes = [] - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): for mon in [r for r in roles if teuthology.is_type('mon', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(mon) @@ -982,7 +996,7 @@ def ceph_mgrs(ctx, config): try: nodes = [] daemons = {} - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): for mgr in [r for r in roles if teuthology.is_type('mgr', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(mgr) @@ -1027,7 +1041,7 @@ def ceph_osds(ctx, config): # provision OSDs in numeric order id_to_remote = {} devs_by_remote = {} - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): devs_by_remote[remote] = teuthology.get_scratch_devices(remote) for osd in [r for r in roles if teuthology.is_type('osd', cluster_name)(r)]: @@ -1111,7 +1125,7 @@ def ceph_mdss(ctx, config): nodes = [] daemons = {} - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): for role in [r for r in roles if teuthology.is_type('mds', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(role) @@ -1188,7 +1202,7 @@ def ceph_monitoring(daemon_type, ctx, config): nodes = [] daemons = {} - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): for role in [r for r in roles if teuthology.is_type(daemon_type, cluster_name)(r)]: c_, _, id_ = teuthology.split_role(role) @@ -1224,7 +1238,7 @@ def ceph_rgw(ctx, config): nodes = {} daemons = {} - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): for role in [r for r in roles if teuthology.is_type('rgw', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(role) @@ -1267,7 +1281,7 @@ def ceph_iscsi(ctx, config): daemons = {} ips = [] - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): for role in [r for r in roles if teuthology.is_type('iscsi', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(role) @@ -1619,7 +1633,7 @@ def distribute_config_and_admin_keyring(ctx, config): """ cluster_name = config['cluster'] log.info('Distributing (final) config and client.admin keyring...') - for remote, roles in ctx.cluster.remotes.items(): + for remote, roles in _cephadm_remotes(ctx): remote.write_file( '/etc/ceph/{}.conf'.format(cluster_name), ctx.ceph[cluster_name].config_file, @@ -1712,7 +1726,7 @@ def initialize_config(ctx, config): # mon ips log.info('Choosing monitor IPs and ports...') - remotes_and_roles = ctx.cluster.remotes.items() + remotes_and_roles = _cephadm_remotes(ctx) ips = [host for (host, port) in (remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)] From 1ed66542ef27f5382d82b7cc60e76c2fe6d30fc9 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Mon, 26 Feb 2024 16:16:57 -0500 Subject: [PATCH 23/29] qa/tasks: a new cephadm exec task similar to vip.exec but generalized Add a new cephadm.exec task that works similarly to the existing vip.exec but instead of only considering VIP related string replacements it uses that templating feature that was recently added to the cephadm module for generalized string templating. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index a7069c19fe7b1..ca6e33d13169d 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -1463,6 +1463,41 @@ def _shell_command(obj): raise ValueError(f'invalid command item: {obj!r}') +def exec(ctx, config): + """ + This is similar to the standard 'exec' task, but does template substitutions. + + TODO: this should probably be moved out of cephadm.py as it's pretty generic. + """ + assert isinstance(config, dict), "task exec got invalid config" + + testdir = teuthology.get_testdir(ctx) + + if 'all-roles' in config and len(config) == 1: + a = config['all-roles'] + roles = teuthology.all_roles(ctx.cluster) + config = dict((id_, a) for id_ in roles if not id_.startswith('host.')) + elif 'all-hosts' in config and len(config) == 1: + a = config['all-hosts'] + roles = teuthology.all_roles(ctx.cluster) + config = dict((id_, a) for id_ in roles if id_.startswith('host.')) + + for role, ls in config.items(): + (remote,) = ctx.cluster.only(role).remotes.keys() + log.info('Running commands on role %s host %s', role, remote.name) + for c in ls: + c.replace('$TESTDIR', testdir) + remote.run( + args=[ + 'sudo', + 'TESTDIR={tdir}'.format(tdir=testdir), + 'bash', + '-ex', + '-c', + _template_transform(ctx, config, c)], + ) + + def apply(ctx, config): """ Apply spec From 361cbd46b9c5431452c742edd3ca7f0280e4161b Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Mon, 26 Feb 2024 16:17:22 -0500 Subject: [PATCH 24/29] qa/tasks: add a template filter to map a role name to a remote Add a `role_to_remote` template filter function that has the ability to map a role name to a remote. Attributes of the remote can then be used to get the actual node ip or name. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index ca6e33d13169d..69510c645e3a1 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -75,6 +75,7 @@ def _template_transform(ctx, config, target): if jenv is None: loader = jinja2.BaseLoader() jenv = jinja2.Environment(loader=loader) + jenv.filters['role_to_remote'] = _role_to_remote setattr(ctx, '_jinja_env', jenv) rctx = dict(ctx=ctx, config=config, cluster_name=config.get('cluster', '')) _vip_vars(rctx) @@ -94,6 +95,16 @@ def _vip_vars(rctx): rctx[f'VIP{idx}'] = str(vip) +@jinja2.pass_context +def _role_to_remote(rctx, role): + """Return the first remote matching the given role.""" + ctx = rctx['ctx'] + for remote, roles in ctx.cluster.remotes.items(): + if role in roles: + return remote + return None + + def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs): teuthology.get_testdir(ctx) return remote.run( From bf1607a4a14e92a745cd8d7e743e5f81b7d407b7 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Tue, 27 Feb 2024 09:44:51 -0500 Subject: [PATCH 25/29] qa/tasks: reduce duplicated code All `exec`-style function in teuthology appear to have a transformation block that expands names like `all-roles` and `all-hosts`. With the new cephadm.exec task that block appeared twice in cephadm.py. This change removes the duplication by creating an _expand_roles function that can be called from the command executing functions. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 69510c645e3a1..fdc9fdcd7392f 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -1420,6 +1420,18 @@ def stop(ctx, config): yield +def _expand_roles(ctx, config): + if 'all-roles' in config and len(config) == 1: + a = config['all-roles'] + roles = teuthology.all_roles(ctx.cluster) + config = dict((id_, a) for id_ in roles if not id_.startswith('host.')) + elif 'all-hosts' in config and len(config) == 1: + a = config['all-hosts'] + roles = teuthology.all_roles(ctx.cluster) + config = dict((id_, a) for id_ in roles if id_.startswith('host.')) + return config + + def shell(ctx, config): """ Execute (shell) commands @@ -1432,15 +1444,7 @@ def shell(ctx, config): for k in config.pop('volumes', []): args.extend(['-v', k]) - if 'all-roles' in config and len(config) == 1: - a = config['all-roles'] - roles = teuthology.all_roles(ctx.cluster) - config = dict((id_, a) for id_ in roles if not id_.startswith('host.')) - elif 'all-hosts' in config and len(config) == 1: - a = config['all-hosts'] - roles = teuthology.all_roles(ctx.cluster) - config = dict((id_, a) for id_ in roles if id_.startswith('host.')) - + config = _expand_roles(ctx, config) config = _template_transform(ctx, config, config) for role, cmd in config.items(): (remote,) = ctx.cluster.only(role).remotes.keys() @@ -1481,18 +1485,8 @@ def exec(ctx, config): TODO: this should probably be moved out of cephadm.py as it's pretty generic. """ assert isinstance(config, dict), "task exec got invalid config" - testdir = teuthology.get_testdir(ctx) - - if 'all-roles' in config and len(config) == 1: - a = config['all-roles'] - roles = teuthology.all_roles(ctx.cluster) - config = dict((id_, a) for id_ in roles if not id_.startswith('host.')) - elif 'all-hosts' in config and len(config) == 1: - a = config['all-hosts'] - roles = teuthology.all_roles(ctx.cluster) - config = dict((id_, a) for id_ in roles if id_.startswith('host.')) - + config = _expand_roles(ctx, config) for role, ls in config.items(): (remote,) = ctx.cluster.only(role).remotes.keys() log.info('Running commands on role %s host %s', role, remote.name) From 96704903f273ca0ee597dd819d9aadd1616625ed Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Tue, 27 Feb 2024 09:48:25 -0500 Subject: [PATCH 26/29] qa/tasks: add error condition to exec functions Looking at the code that expands `all-roles` and `all-hosts` there's no proper error checking for when these values appear but there are >1 top-level roles in the task config. If a user does this it'll fail but in a somewhat unclear manner. Add a new condition that raises a clear exception in this case hopefully saving someone future debugging time. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index fdc9fdcd7392f..66d51aff2f548 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -1429,6 +1429,10 @@ def _expand_roles(ctx, config): a = config['all-hosts'] roles = teuthology.all_roles(ctx.cluster) config = dict((id_, a) for id_ in roles if id_.startswith('host.')) + elif 'all-roles' in config or 'all-hosts' in config: + raise ValueError( + 'all-roles/all-hosts may not be combined with any other roles' + ) return config From 1f3001eef670a6bd1ff47cc11e459058a523b388 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Sat, 24 Feb 2024 10:52:53 -0500 Subject: [PATCH 27/29] qa/suites/orch: add a new smb service cephadm sub-suite and test Start a new subdir under cephadm suite for the new smb service that cephadm can deploy. Add one new test that checks that a smb service with domain membership can be deployed and connect to it with smbclient from the samba client container image. Signed-off-by: John Mulligan --- qa/suites/orch/cephadm/smb/% | 0 qa/suites/orch/cephadm/smb/.qa | 1 + qa/suites/orch/cephadm/smb/0-distro | 1 + qa/suites/orch/cephadm/smb/tasks/.qa | 1 + .../cephadm/smb/tasks/deploy_smb_domain.yaml | 88 +++++++++++++++++++ 5 files changed, 91 insertions(+) create mode 100644 qa/suites/orch/cephadm/smb/% create mode 120000 qa/suites/orch/cephadm/smb/.qa create mode 120000 qa/suites/orch/cephadm/smb/0-distro create mode 120000 qa/suites/orch/cephadm/smb/tasks/.qa create mode 100644 qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml diff --git a/qa/suites/orch/cephadm/smb/% b/qa/suites/orch/cephadm/smb/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/orch/cephadm/smb/.qa b/qa/suites/orch/cephadm/smb/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/orch/cephadm/smb/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/orch/cephadm/smb/0-distro b/qa/suites/orch/cephadm/smb/0-distro new file mode 120000 index 0000000000000..66187855738ef --- /dev/null +++ b/qa/suites/orch/cephadm/smb/0-distro @@ -0,0 +1 @@ +.qa/distros/supported-container-hosts \ No newline at end of file diff --git a/qa/suites/orch/cephadm/smb/tasks/.qa b/qa/suites/orch/cephadm/smb/tasks/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/orch/cephadm/smb/tasks/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml new file mode 100644 index 0000000000000..7662c9d6a468e --- /dev/null +++ b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml @@ -0,0 +1,88 @@ +roles: +# Test is for basic smb deployment & functionality. one node cluster is OK +- - host.a + - mon.a + - mgr.x + - osd.0 + - osd.1 + - client.0 +# Reserve a host for acting as a domain controller +- - host.b + - cephadm.exclude +tasks: +- cephadm.deploy_samba_ad_dc: + role: host.b +- cephadm: + +- cephadm.shell: + host.a: + - ceph fs volume create cephfs +- cephadm.wait_for_service: + service: mds.cephfs + +- cephadm.shell: + host.a: + # create a subvolume so we can verify that we're sharing something + - cmd: ceph fs subvolumegroup create cephfs g1 + - cmd: ceph fs subvolume create cephfs sub1 --group-name=g1 --mode=0777 + # Create a user access the file system from samba + - cmd: ceph fs authorize cephfs client.smbdata / rw + # Create a rados pool and store the config in it + - cmd: ceph osd pool create .smb --yes-i-really-mean-it + - cmd: ceph osd pool application enable .smb smb + - cmd: rados --pool=.smb --namespace=admem1 put conf.toml /dev/stdin + stdin: | + samba-container-config = "v0" + [configs.admem1] + shares = ["share1"] + globals = ["default", "domain"] + instance_name = "SAMBA" + [shares.share1.options] + "vfs objects" = "ceph" + path = "/" + "ceph:config_file" = "/etc/ceph/ceph.conf" + "ceph:user_id" = "smbdata" + "kernel share modes" = "no" + "read only" = "no" + "browseable" = "yes" + [globals.default.options] + "server min protocol" = "SMB2" + "load printers" = "no" + "printing" = "bsd" + "printcap name" = "/dev/null" + "disable spoolss" = "yes" + "guest ok" = "no" + [globals.domain.options] + security = "ads" + workgroup = "DOMAIN1" + realm = "domain1.sink.test" + "idmap config * : backend" = "autorid" + "idmap config * : range" = "2000-9999999" + # Store the join auth user/pass in the config-key store + - cmd: ceph config-key set smb/config/admem1/join1.json -i - + stdin: | + {"username": "Administrator", "password": "Passw0rd"} + +- cephadm.apply: + specs: + - service_type: smb + service_id: admem1 + placement: + count: 1 + cluster_id: admem1 + features: + - domain + config_uri: "rados://.smb/admem1/conf.toml" + custom_dns: + - "{{ctx.samba_ad_dc_ip}}" + join_sources: + - "rados:mon-config-key:smb/config/admem1/join1.json" + include_ceph_users: + - "client.smbdata" +- cephadm.wait_for_service: + service: smb.admem1 + +- cephadm.exec: + host.b: + - sleep 30 + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share1 -c ls" From b2197e43b5eeb326b2e5498a06a2e13f4a532b87 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 15 Mar 2024 13:48:35 -0400 Subject: [PATCH 28/29] qa/tasks: add a cephadm samba container helper func independent of AD DC To have the standalone (non-AD) server test function similarly to the AD member server test we need to set a variable for samba client container command similar to how the AD setup command does it. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 66d51aff2f548..008d8c94d55bb 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -1962,6 +1962,44 @@ def _samba_ad_dc_conf(ctx, remote, cengine): ] +@contextlib.contextmanager +def configure_samba_client_container(ctx, config): + # TODO: deduplicate logic between this task and deploy_samba_ad_dc + role = config.get('role') + samba_client_image = config.get( + 'samba_client_image', 'quay.io/samba.org/samba-client:latest' + ) + if not role: + raise ConfigError( + "you must specify a role to discover container engine / pull image" + ) + (remote,) = ctx.cluster.only(role).remotes.keys() + cengine = 'podman' + try: + log.info("Testing if podman is available") + remote.run(args=['sudo', cengine, '--help']) + except CommandFailedError: + log.info("Failed to find podman. Using docker") + cengine = 'docker' + + remote.run(args=['sudo', cengine, 'pull', samba_client_image]) + samba_client_container_cmd = [ + 'sudo', + cengine, + 'run', + '--rm', + '--net=host', + '-eKRB5_CONFIG=/dev/null', + samba_client_image, + ] + + setattr(ctx, 'samba_client_container_cmd', samba_client_container_cmd) + try: + yield + finally: + setattr(ctx, 'samba_client_container_cmd', None) + + @contextlib.contextmanager def deploy_samba_ad_dc(ctx, config): role = config.get('role') From 8bb5fb69648f497da80c97011e171dff23c5130d Mon Sep 17 00:00:00 2001 From: Shachar Sharon Date: Wed, 13 Mar 2024 16:43:29 +0200 Subject: [PATCH 29/29] qa/suites/orch: add minimal smb non-AD test Test minimal SMB deployment over CephFS, using local users (non-AD). Upon successful deployment run minima smbclient command ('ls') to probe Samba's share liveness. Co-authored-by: John Mulligan Signed-off-by: Shachar Sharon Signed-off-by: John Mulligan --- .../cephadm/smb/tasks/deploy_smb_basic.yaml | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml new file mode 100644 index 0000000000000..a64591ce8f82a --- /dev/null +++ b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml @@ -0,0 +1,77 @@ +--- +roles: + # Test is for basic smb deployment & functionality. one node cluster is OK + - - host.a + - mon.a + - mgr.x + - osd.0 + - osd.1 + - client.0 + # Reserve a host for acting as a test client + - - host.b + - cephadm.exclude +tasks: + # TODO: (jjm) I don't think `install` is necessary for this file. Remove? + - install: + - cephadm.configure_samba_client_container: + role: host.b + - cephadm: + - cephadm.shell: + host.a: + - ceph fs volume create cephfs + - cephadm.wait_for_service: + service: mds.cephfs + - cephadm.shell: + host.a: + # create a subvolume so we can verify that we're sharing something + - cmd: ceph fs subvolumegroup create cephfs g1 + - cmd: ceph fs subvolume create cephfs sub1 --group-name=g1 --mode=0777 + # Create a user access the file system from samba + - cmd: ceph fs authorize cephfs client.smbdata / rw + # Create a rados pool and store the config in it + - cmd: ceph osd pool create .smb --yes-i-really-mean-it + - cmd: ceph osd pool application enable .smb smb + - cmd: rados --pool=.smb --namespace=saserv1 put conf.toml /dev/stdin + stdin: | + samba-container-config = "v0" + [configs.saserv1] + shares = ["share1"] + globals = ["default", "domain"] + instance_name = "SAMBA" + [shares.share1.options] + "vfs objects" = "ceph" + path = "/" + "ceph:config_file" = "/etc/ceph/ceph.conf" + "ceph:user_id" = "smbdata" + "kernel share modes" = "no" + "read only" = "no" + "browseable" = "yes" + [globals.default.options] + "server min protocol" = "SMB2" + "load printers" = "no" + "printing" = "bsd" + "printcap name" = "/dev/null" + "disable spoolss" = "yes" + "guest ok" = "no" + [globals.domain.options] + security = "USER" + workgroup = "STANDALONE1" + [[users.all_entries]] + name = "smbuser1" + password = "insecure321" + - cephadm.apply: + specs: + - service_type: smb + service_id: saserv1 + placement: + count: 1 + cluster_id: saserv1 + config_uri: "rados://.smb/saserv1/conf.toml" + include_ceph_users: + - "client.smbdata" + - cephadm.wait_for_service: + service: smb.saserv1 + - cephadm.exec: + host.b: + - sleep 30 + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U smbuser1%insecure321 //{{'host.a'|role_to_remote|attr('ip_address')}}/share1 -c ls"