mgr/cephadm: automatically configure dashboard <-> mgr connection

Automatically configure the dashboard creds to talk to RGW.

Fixes: https://tracker.ceph.com/issues/44605
Signed-off-by: Sage Weil <sage@newdream.net>

src/pybind/mgr/cephadm/module.py

@@ -466,6 +466,7 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.template = TemplateMgr(self)
 
         self.requires_post_actions: Set[str] = set()
+        self.need_connect_dashboard_rgw = False
 
         self.config_checker = CephadmConfigChecks(self)
 
@@ -2546,3 +2547,7 @@ def remove_osds_status(self) -> List[OSD]:
         The CLI call to retrieve an osd removal report
         """
         return self.to_remove_osds.all_osds()
+
+    def trigger_connect_dashboard_rgw(self) -> None:
+        self.need_connect_dashboard_rgw = True
+        self.event.set()

src/pybind/mgr/cephadm/serve.py

@@ -22,7 +22,7 @@
 import orchestrator
 from orchestrator import OrchestratorError, set_exception_subject, OrchestratorEvent, \
     DaemonDescriptionStatus, daemon_type_to_service
-from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
+from cephadm.services.cephadmservice import CephadmDaemonDeploySpec, RgwService
 from cephadm.schedule import HostAssignment
 from cephadm.autotune import MemoryAutotuner
 from cephadm.utils import forall_hosts, cephadmNoImage, is_repo_digest, \
@@ -38,6 +38,8 @@
 
 logger = logging.getLogger(__name__)
 
+REQUIRES_POST_ACTIONS = ['grafana', 'iscsi', 'prometheus', 'alertmanager', 'rgw']
+
 
 class CephadmServe:
     """
@@ -79,6 +81,11 @@ def serve(self) -> None:
 
                 self._update_paused_health()
 
+                if self.mgr.need_connect_dashboard_rgw and self.mgr.config_dashboard:
+                    self.mgr.need_connect_dashboard_rgw = False
+                    if 'dashboard' in self.mgr.get('mgr_map')['modules']:
+                        cast(RgwService, self.mgr.cephadm_services['rgw']).connect_dashboard_rgw()
+
                 if not self.mgr.paused:
                     self.mgr.to_remove_osds.process_removal_queue()
 
@@ -878,7 +885,7 @@ def _check_daemons(self) -> None:
                 continue
 
             # These daemon types require additional configs after creation
-            if dd.daemon_type in ['grafana', 'iscsi', 'prometheus', 'alertmanager', 'nfs']:
+            if dd.daemon_type in REQUIRES_POST_ACTIONS:
                 daemons_post[dd.daemon_type].append(dd)
 
             if self.mgr.cephadm_services[daemon_type_to_service(dd.daemon_type)].get_active_daemon(
@@ -1055,9 +1062,7 @@ def _create_daemon(self,
                         sd = daemon_spec.to_daemon_description(
                             DaemonDescriptionStatus.running, 'starting')
                         self.mgr.cache.add_daemon(daemon_spec.host, sd)
-                        if daemon_spec.daemon_type in [
-                            'grafana', 'iscsi', 'prometheus', 'alertmanager'
-                        ]:
+                        if daemon_spec.daemon_type in REQUIRES_POST_ACTIONS:
                             self.mgr.requires_post_actions.add(daemon_spec.daemon_type)
                     self.mgr.cache.invalidate_host_daemons(daemon_spec.host)
 

src/pybind/mgr/cephadm/services/cephadmservice.py

@@ -1,7 +1,9 @@
+import datetime
 import errno
 import json
 import logging
 import re
+import subprocess
 from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING, List, Callable, TypeVar, \
     Optional, Dict, Any, Tuple, NewType, cast
@@ -10,7 +12,7 @@
 
 from ceph.deployment.service_spec import ServiceSpec, RGWSpec
 from ceph.deployment.utils import is_ipv6, unwrap_ipv6
-from orchestrator import OrchestratorError, DaemonDescription, DaemonDescriptionStatus
+from orchestrator import OrchestratorError, DaemonDescription, DaemonDescriptionStatus, raise_if_exception
 from orchestrator._interface import daemon_type_to_service
 from cephadm import utils
 
@@ -882,6 +884,7 @@ def post_remove(self, daemon: DaemonDescription) -> None:
             'who': utils.name_to_config_section(daemon.name()),
             'name': 'rgw_frontends',
         })
+        self.mgr.trigger_connect_dashboard_rgw()
 
     def ok_to_stop(
             self,
@@ -918,6 +921,114 @@ def ingress_present() -> bool:
         warn_message = "WARNING: Removing RGW daemons can cause clients to lose connectivity. "
         return HandleCommandResult(-errno.EBUSY, '', warn_message)
 
+    def config_dashboard(self, daemon_descrs: List[DaemonDescription]) -> None:
+        self.mgr.trigger_connect_dashboard_rgw()
+
+    def connect_dashboard_rgw(self) -> None:
+        """
+        Configure the dashboard to talk to RGW
+        """
+        self.mgr.log.info('Checking dashboard <-> RGW connection')
+        try:
+            self._connect_dashboard_rgw()
+        except Exception as e:
+            self.mgr.log.error(f'Failed to configure dashboard <-> RGW connection: {e}')
+
+    def _connect_dashboard_rgw(self) -> None:
+        def radosgw_admin(args: List[str]) -> Tuple[int, str, str]:
+            try:
+                result = subprocess.run(
+                    [
+                        'radosgw-admin',
+                        '-c', str(self.mgr.get_ceph_conf_path()),
+                        '-k', str(self.mgr.get_ceph_option('keyring')),
+                        '-n', f'mgr.{self.mgr.get_mgr_id()}',
+                    ] + args,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    timeout=10,
+                )
+                return result.returncode, result.stdout.decode('utf-8'), result.stderr.decode('utf-8')
+            except subprocess.CalledProcessError as ex:
+                self.mgr.log.error(f'Error executing radosgw-admin {ex.cmd}: {ex.output}')
+                raise
+            except subprocess.TimeoutExpired as ex:
+                self.mgr.log.error(f'Timeout (10s) executing radosgw-admin {ex.cmd}')
+                raise
+
+        def get_secrets(user: str, out: str) -> Tuple[Optional[str], Optional[str]]:
+            r = json.loads(out)
+            for k in r.get('keys', []):
+                if k.get('user') == user and r.get('system') in ['true', True]:
+                    access_key = k.get('access_key')
+                    secret_key = k.get('secret_key')
+                    return access_key, secret_key
+            return None, None
+
+        def update_dashboard(what: str, value: str) -> None:
+            _, out, _ = self.mgr.check_mon_command({'prefix': f'dashboard get-{what}'})
+            if out.strip() != value:
+                if what.endswith('-key'):
+                    self.mgr.check_mon_command(
+                        {'prefix': f'dashboard set-{what}'},
+                        inbuf=value
+                    )
+                else:
+                    self.mgr.check_mon_command({'prefix': f'dashboard set-{what}',
+                                                "value": value})
+                self.mgr.log.info(f'Updated dashboard {what}')
+
+        completion = self.mgr.list_daemons(daemon_type='rgw')
+        raise_if_exception(completion)
+        daemons = completion.result
+        if not daemons:
+            self.mgr.log.info('No remaining RGW daemons; disconnecting dashboard')
+            self.mgr.check_mon_command({'prefix': 'dashboard reset-rgw-api-host'})
+            self.mgr.check_mon_command({'prefix': 'dashboard reset-rgw-api-port'})
+            self.mgr.check_mon_command({'prefix': 'dashboard reset-rgw-api-scheme'})
+            return
+
+        ret, out, err = radosgw_admin('realm', 'list')
+        if ret:
+            self.mgr.log.error(f'Failed to list RGW realms: {err}')
+            return
+        realms = json.loads(out).get('realms', [])
+
+        def setup_user(realm: Optional[str]) -> Tuple[str, str]:
+            user = f'dashboard-{realm}' if realm else 'dashboard'
+            access_key = None
+            secret_key = None
+            rc, out, _ = radosgw_admin(['user', 'info', '--uid', user])
+            if not rc:
+                access_key, secret_key = get_secrets(user, out)
+                if not access_key:
+                    rc, out, err = radosgw_admin([
+                        'user', 'create', '--uid', user,
+                        '--display-name', 'Ceph Dashboard {realm}',
+                        '--system',
+                    ])
+                if not rc:
+                    access_key, secret_key = get_secrets(user, out)
+                    if not access_key:
+                        self.mgr.log.error(f'Unable to create rgw {user} user: {err}')
+            return access_key, secret_key
+
+        if realms:
+            # multisite! we need creds for each realm
+            a: Dict[str, str] = {}
+            s: Dict[str, str] = {}
+            for realm in realms:
+                a[realm], s[realm] = setup_user(realm)
+            update_dashboard('rgw-api-access-key', json.dump(a))
+            update_dashboard('rgw-api-secret-key', json.dump(s))
+        else:
+            # legacy (no multisite)
+            access_key, secret_key = setup_user(None)
+            if not access_key:
+                return
+            update_dashboard('rgw-api-access-key', access_key)
+            update_dashboard('rgw-api-secret-key', secret_key)
+
 
 class RbdMirrorService(CephService):
     TYPE = 'rbd-mirror'