mgr/dashboard: Implement drain host functionality in dashboard

Fixes: https://tracker.ceph.com/issues/51587
Signed-off-by: Nizamudeen A <nia@redhat.com>

src/pybind/mgr/dashboard/ci/cephadm/ceph_cluster.yml

@@ -1,5 +1,5 @@
 parameters:
- nodes: 3
+ nodes: 4
  pool: ceph-dashboard
  network: ceph-dashboard
  domain: cephlab.com

src/pybind/mgr/dashboard/controllers/host.py

@@ -419,21 +419,23 @@ def get(self, hostname: str) -> Dict:
     @raise_if_no_orchestrator([OrchFeature.HOST_LABEL_ADD,
                                OrchFeature.HOST_LABEL_REMOVE,
                                OrchFeature.HOST_MAINTENANCE_ENTER,
-                               OrchFeature.HOST_MAINTENANCE_EXIT])
+                               OrchFeature.HOST_MAINTENANCE_EXIT,
+                               OrchFeature.HOST_DRAIN])
     @handle_orchestrator_error('host')
     @EndpointDoc('',
                  parameters={
                      'hostname': (str, 'Hostname'),
                      'update_labels': (bool, 'Update Labels'),
                      'labels': ([str], 'Host Labels'),
                      'maintenance': (bool, 'Enter/Exit Maintenance'),
-                     'force': (bool, 'Force Enter Maintenance')
+                     'force': (bool, 'Force Enter Maintenance'),
+                     'drain': (bool, 'Drain Host')
                  },
                  responses={200: None, 204: None})
     @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL)
     def set(self, hostname: str, update_labels: bool = False,
             labels: List[str] = None, maintenance: bool = False,
-            force: bool = False):
+            force: bool = False, drain: bool = False):
         """
         Update the specified host.
         Note, this is only supported when Ceph Orchestrator is enabled.
@@ -442,6 +444,7 @@ def set(self, hostname: str, update_labels: bool = False,
         :param labels: List of labels.
         :param maintenance: Enter/Exit maintenance mode.
         :param force: Force enter maintenance mode.
+        :param drain: Drain host
         """
         orch = OrchClient.instance()
         host = get_host(hostname)
@@ -454,6 +457,9 @@ def set(self, hostname: str, update_labels: bool = False,
             if status == 'maintenance':
                 orch.hosts.exit_maintenance(hostname)
 
+        if drain:
+            orch.hosts.drain(hostname)
+
         if update_labels:
             # only allow List[str] type for labels
             if not isinstance(labels, list):

src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts

@@ -80,7 +80,7 @@ export class HostsPageHelper extends PageHelper {
     });
   }
 
-  delete(hostname: string) {
+  remove(hostname: string) {
     super.delete(hostname, this.columnIndex.hostname, 'hosts');
   }
 
@@ -173,4 +173,17 @@ export class HostsPageHelper extends PageHelper {
         });
     }
   }
+
+  @PageHelper.restrictTo(pages.index.url)
+  drain(hostname: string) {
+    this.getTableCell(this.columnIndex.hostname, hostname).click();
+    this.clickActionButton('start-drain');
+    this.checkLabelExists(hostname, ['_no_schedule'], true);
+
+    this.clickTab('cd-host-details', hostname, 'Daemons');
+    cy.get('cd-host-details').within(() => {
+      cy.wait(10000);
+      this.expectTableCount('total', 0);
+    });
+  }
 }

src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts

@@ -21,21 +21,12 @@ describe('Hosts page', () => {
       hosts.add(hostname, true);
     });
 
-    it('should drain and delete a host and then add it back', function () {
+    it('should drain and remove a host and then add it back', function () {
       const hostname = Cypress._.last(this.hosts)['name'];
 
       // should drain the host first before deleting
-      hosts.editLabels(hostname, ['_no_schedule'], true);
-      hosts.clickTab('cd-host-details', hostname, 'Daemons');
-      cy.get('cd-host-details').within(() => {
-        // draining will take some time to complete.
-        // since we don't know how many daemons will be
-        // running in this host in future putting the wait
-        // to 15s
-        cy.wait(15000);
-        hosts.getTableCount('total').should('be.eq', 0);
-      });
-      hosts.delete(hostname);
+      hosts.drain(hostname);
+      hosts.remove(hostname);
 
       // add it back
       hosts.navigateTo('add');

src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts

@@ -10,7 +10,7 @@ describe('Create cluster add host page', () => {
     'ceph-node-00.cephlab.com',
     'ceph-node-01.cephlab.com',
     'ceph-node-02.cephlab.com',
-    'ceph-node-[01-02].cephlab.com'
+    'ceph-node-[01-03].cephlab.com'
   ];
   const addHost = (hostname: string, exist?: boolean, pattern?: boolean, labels: string[] = []) => {
     cy.get('.btn.btn-accent').first().click({ force: true });
@@ -38,13 +38,13 @@ describe('Create cluster add host page', () => {
 
     addHost(hostnames[1], false);
     addHost(hostnames[2], false);
-    createClusterHostPage.delete(hostnames[1]);
-    createClusterHostPage.delete(hostnames[2]);
+    createClusterHostPage.remove(hostnames[1]);
+    createClusterHostPage.remove(hostnames[2]);
     addHost(hostnames[3], false, true);
   });
 
-  it('should delete a host', () => {
-    createClusterHostPage.delete(hostnames[1]);
+  it('should remove a host', () => {
+    createClusterHostPage.remove(hostnames[1]);
   });
 
   it('should add a host with some predefined labels and verify it', () => {

src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts

@@ -27,7 +27,8 @@ describe('when cluster creation is completed', () => {
     const hostnames = [
       'ceph-node-00.cephlab.com',
       'ceph-node-01.cephlab.com',
-      'ceph-node-02.cephlab.com'
+      'ceph-node-02.cephlab.com',
+      'ceph-node-03.cephlab.com'
     ];
 
     beforeEach(() => {
@@ -55,14 +56,22 @@ describe('when cluster creation is completed', () => {
     });
 
     it('should check if rgw service is running', () => {
-      hosts.clickTab('cd-host-details', hostnames[1], 'Daemons');
+      hosts.clickTab('cd-host-details', hostnames[3], 'Daemons');
       cy.get('cd-host-details').within(() => {
         services.checkServiceStatus('rgw');
       });
     });
 
     it('should force maintenance and exit', { retries: 1 }, () => {
-      hosts.maintenance(hostnames[1], true, true);
+      hosts.maintenance(hostnames[3], true, true);
+    });
+
+    it('should drain, remove and add the host back', () => {
+      hosts.drain(hostnames[1]);
+      hosts.remove(hostnames[1]);
+      hosts.navigateTo('add');
+      hosts.add(hostnames[1]);
+      hosts.checkExist(hostnames[1], true);
     });
   });
 

src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts

@@ -294,7 +294,8 @@ describe('HostsComponent', () => {
         OrchestratorFeature.HOST_ADD,
         OrchestratorFeature.HOST_LABEL_ADD,
         OrchestratorFeature.HOST_REMOVE,
-        OrchestratorFeature.HOST_LABEL_REMOVE
+        OrchestratorFeature.HOST_LABEL_REMOVE,
+        OrchestratorFeature.HOST_DRAIN
       ];
       await testTableActions(true, features, tests);
     });

src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts

@@ -84,7 +84,8 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
   modalRef: NgbModalRef;
   isExecuting = false;
   errorMessage: string;
-  enableButton: boolean;
+  enableMaintenanceBtn: boolean;
+  enableDrainBtn: boolean;
   bsModalRef: NgbModalRef;
 
   icons = Icons;
@@ -101,7 +102,8 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
     maintenance: [
       OrchestratorFeature.HOST_MAINTENANCE_ENTER,
       OrchestratorFeature.HOST_MAINTENANCE_EXIT
-    ]
+    ],
+    drain: [OrchestratorFeature.HOST_DRAIN]
   };
 
   constructor(
@@ -135,6 +137,24 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
         click: () => this.editAction(),
         disable: (selection: CdTableSelection) => this.getDisable('edit', selection)
       },
+      {
+        name: this.actionLabels.START_DRAIN,
+        permission: 'update',
+        icon: Icons.exit,
+        click: () => this.hostDrain(),
+        disable: (selection: CdTableSelection) =>
+          this.getDisable('drain', selection) || !this.enableDrainBtn,
+        visible: () => !this.showGeneralActionsOnly && this.enableDrainBtn
+      },
+      {
+        name: this.actionLabels.STOP_DRAIN,
+        permission: 'update',
+        icon: Icons.exit,
+        click: () => this.hostDrain(true),
+        disable: (selection: CdTableSelection) =>
+          this.getDisable('drain', selection) || this.enableDrainBtn,
+        visible: () => !this.showGeneralActionsOnly && !this.enableDrainBtn
+      },
       {
         name: this.actionLabels.REMOVE,
         permission: 'delete',
@@ -148,17 +168,21 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
         icon: Icons.enter,
         click: () => this.hostMaintenance(),
         disable: (selection: CdTableSelection) =>
-          this.getDisable('maintenance', selection) || this.isExecuting || this.enableButton,
-        visible: () => !this.showGeneralActionsOnly
+          this.getDisable('maintenance', selection) ||
+          this.isExecuting ||
+          this.enableMaintenanceBtn,
+        visible: () => !this.showGeneralActionsOnly && !this.enableMaintenanceBtn
       },
       {
         name: this.actionLabels.EXIT_MAINTENANCE,
         permission: 'update',
         icon: Icons.exit,
         click: () => this.hostMaintenance(),
         disable: (selection: CdTableSelection) =>
-          this.getDisable('maintenance', selection) || this.isExecuting || !this.enableButton,
-        visible: () => !this.showGeneralActionsOnly
+          this.getDisable('maintenance', selection) ||
+          this.isExecuting ||
+          !this.enableMaintenanceBtn,
+        visible: () => !this.showGeneralActionsOnly && this.enableMaintenanceBtn
       }
     ];
   }
@@ -252,10 +276,15 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
 
   updateSelection(selection: CdTableSelection) {
     this.selection = selection;
-    this.enableButton = false;
+    this.enableMaintenanceBtn = false;
+    this.enableDrainBtn = false;
     if (this.selection.hasSelection) {
       if (this.selection.first().status === 'maintenance') {
-        this.enableButton = true;
+        this.enableMaintenanceBtn = true;
+      }
+
+      if (!this.selection.first().labels.includes('_no_schedule')) {
+        this.enableDrainBtn = true;
       }
     }
   }
@@ -361,11 +390,39 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
     }
   }
 
+  hostDrain(stop = false) {
+    const host = this.selection.first();
+    if (stop) {
+      const index = host['labels'].indexOf('_no_schedule', 0);
+      host['labels'].splice(index, 1);
+      this.hostService.update(host['hostname'], true, host['labels']).subscribe(() => {
+        this.notificationService.show(
+          NotificationType.info,
+          $localize`"${host['hostname']}" stopped draining`
+        );
+        this.table.refreshBtn();
+      });
+    } else {
+      this.hostService.update(host['hostname'], false, [], false, false, true).subscribe(() => {
+        this.notificationService.show(
+          NotificationType.info,
+          $localize`"${host['hostname']}" started draining`
+        );
+        this.table.refreshBtn();
+      });
+    }
+  }
+
   getDisable(
-    action: 'add' | 'edit' | 'remove' | 'maintenance',
+    action: 'add' | 'edit' | 'remove' | 'maintenance' | 'drain',
     selection: CdTableSelection
   ): boolean | string {
-    if (action === 'remove' || action === 'edit' || action === 'maintenance') {
+    if (
+      action === 'remove' ||
+      action === 'edit' ||
+      action === 'maintenance' ||
+      action === 'drain'
+    ) {
       if (!selection?.hasSingleSelection) {
         return true;
       }

src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts

@@ -44,14 +44,28 @@ describe('HostService', () => {
   });
 
   it('should update host', fakeAsync(() => {
-    service.update('mon0', true, ['foo', 'bar']).subscribe();
+    service.update('mon0', true, ['foo', 'bar'], true, false).subscribe();
     const req = httpTesting.expectOne('api/host/mon0');
     expect(req.request.method).toBe('PUT');
     expect(req.request.body).toEqual({
       force: false,
       labels: ['foo', 'bar'],
+      maintenance: true,
+      update_labels: true,
+      drain: false
+    });
+  }));
+
+  it('should test host drain call', fakeAsync(() => {
+    service.update('host0', false, null, false, false, true).subscribe();
+    const req = httpTesting.expectOne('api/host/host0');
+    expect(req.request.method).toBe('PUT');
+    expect(req.request.body).toEqual({
+      force: false,
+      labels: null,
       maintenance: false,
-      update_labels: true
+      update_labels: false,
+      drain: true
     });
   }));
 

src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts

@@ -69,15 +69,17 @@ export class HostService extends ApiClient {
     updateLabels = false,
     labels: string[] = [],
     maintenance = false,
-    force = false
+    force = false,
+    drain = false
   ) {
     return this.http.put(
       `${this.baseURL}/${hostname}`,
       {
         update_labels: updateLabels,
         labels: labels,
         maintenance: maintenance,
-        force: force
+        force: force,
+        drain: drain
       },
       { headers: { Accept: this.getVersionHeaderValue(0, 1) } }
     );

src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts

@@ -118,6 +118,8 @@ export class ActionLabelsI18n {
   FLAGS: string;
   ENTER_MAINTENANCE: string;
   EXIT_MAINTENANCE: string;
+  START_DRAIN: string;
+  STOP_DRAIN: string;
 
   constructor() {
     /* Create a new item */
@@ -171,6 +173,8 @@ export class ActionLabelsI18n {
     this.FLAGS = $localize`Flags`;
     this.ENTER_MAINTENANCE = $localize`Enter Maintenance`;
     this.EXIT_MAINTENANCE = $localize`Exit Maintenance`;
+    this.START_DRAIN = $localize`Start Drain`;
+    this.STOP_DRAIN = $localize`Stop Drain`;
 
     /* Prometheus wording */
     this.RECREATE = $localize`Recreate`;

src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts

@@ -7,6 +7,7 @@ export enum OrchestratorFeature {
   HOST_MAINTENANCE_ENTER = 'enter_host_maintenance',
   HOST_MAINTENANCE_EXIT = 'exit_host_maintenance',
   HOST_FACTS = 'get_facts',
+  HOST_DRAIN = 'drain_host',
 
   SERVICE_LIST = 'describe_service',
   SERVICE_CREATE = 'apply',