Feature: Safely shutdown cloudstack

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -523,6 +523,7 @@ public class ApiConstants {
     public static final String PRIVATE_NETWORK_ID = "privatenetworkid";
     public static final String ALLOCATION_STATE = "allocationstate";
     public static final String MANAGED_STATE = "managedstate";
+    public static final String MANAGEMENT_SERVER_ID = "managementserverid";
     public static final String STORAGE_ID = "storageid";
     public static final String PING_STORAGE_SERVER_IP = "pingstorageserverip";
     public static final String PING_DIR = "pingdir";
@@ -1018,6 +1019,10 @@ public class ApiConstants {
     public static final String LOGOUT = "logout";
     public static final String LIST_IDPS = "listIdps";
 
+    public static final String READY_FOR_SHUTDOWN = "readyforshutdown";
+    public static final String SHUTDOWN_TRIGGERED = "shutdowntriggered";
+    public static final String PENDING_JOBS_COUNT = "pendingjobscount";
+
     public static final String PUBLIC_MTU = "publicmtu";
     public static final String PRIVATE_MTU = "privatemtu";
     public static final String MTU = "mtu";

api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java

@@ -24,6 +24,7 @@
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.AsyncJobResponse;
 import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.ManagementServerResponse;
 
 @APICommand(name = "listAsyncJobs", description = "Lists all pending asynchronous jobs for the account.", responseObject = AsyncJobResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -36,6 +37,9 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd {
     @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date of the async job (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")")
     private Date startDate;
 
+    @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "The id of the management server", since="4.19")
+    private Long managementServerId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -44,6 +48,10 @@ public Date getStartDate() {
         return startDate;
     }
 
+    public Long getManagementServerId() {
+        return managementServerId;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java

@@ -32,9 +32,21 @@
 public class AsyncJobResponse extends BaseResponse {
 
     @SerializedName("accountid")
-    @Param(description = "the account that executed the async command")
+    @Param(description = "the account id that executed the async command")
     private String accountId;
 
+    @SerializedName("account")
+    @Param(description = "the account that executed the async command")
+    private String account;
+
+    @SerializedName("domainid")
+    @Param(description = "the domain id that executed the async command")
+    private String domainid;
+
+    @SerializedName("domainpath")
+    @Param(description = "the domain that executed the async command")
+    private String domainPath;
+
     @SerializedName(ApiConstants.USER_ID)
     @Param(description = "the user that executed the async command")
     private String userId;
@@ -71,6 +83,10 @@ public class AsyncJobResponse extends BaseResponse {
     @Param(description = "the unique ID of the instance/entity object related to the job")
     private String jobInstanceId;
 
+    @SerializedName("managementserverid")
+    @Param(description = "the msid of the management server on which the job is running", since = "4.19")
+    private Long msid;
+
     @SerializedName(ApiConstants.CREATED)
     @Param(description = "  the created date of the job")
     private Date created;
@@ -83,6 +99,18 @@ public void setAccountId(String accountId) {
         this.accountId = accountId;
     }
 
+    public void setAccount(String account) {
+        this.account = account;
+    }
+
+    public void setDomainId(String domainid) {
+        this.domainid = domainid;
+    }
+
+    public void setDomainPath(String domainPath) {
+        this.domainPath = domainPath;
+    }
+
     public void setUserId(String userId) {
         this.userId = userId;
     }
@@ -127,4 +155,8 @@ public void setCreated(Date created) {
     public void setRemoved(final Date removed) {
         this.removed = removed;
     }
+
+    public void setMsid(Long msid) {
+        this.msid = msid;
+    }
 }

api/src/main/java/org/apache/cloudstack/management/ManagementServerHost.java

@@ -21,7 +21,7 @@
 
 public interface ManagementServerHost extends InternalIdentity, Identity {
     enum State {
-        Up, Down
+        Up, Down, PreparingToShutDown, ReadyToShutDown, ShuttingDown
     }
 
     long getMsid();

client/pom.xml

@@ -552,6 +552,11 @@
             <artifactId>cloud-plugin-integrations-kubernetes-service</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+           <groupId>org.apache.cloudstack</groupId>
+           <artifactId>cloud-plugin-shutdown</artifactId>
+           <version>${project.version}</version>
+       </dependency>
     </dependencies>
     <build>
         <plugins>

engine/orchestration/pom.xml

@@ -68,6 +68,11 @@
             <artifactId>cloud-server</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-shutdown</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -50,6 +50,11 @@
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
 import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
+import org.apache.cloudstack.shutdown.ShutdownManager;
+import org.apache.cloudstack.shutdown.command.CancelShutdownManagementServerHostCommand;
+import org.apache.cloudstack.shutdown.command.PrepareForShutdownManagementServerHostCommand;
+import org.apache.cloudstack.shutdown.command.BaseShutdownManagementServerHostCommand;
+import org.apache.cloudstack.shutdown.command.TriggerShutdownManagementServerHostCommand;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.cloudstack.utils.security.SSLUtils;
 import org.apache.log4j.Logger;
@@ -129,6 +134,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     private HAConfigDao haConfigDao;
     @Inject
     private CAManager caService;
+    @Inject
+    private ShutdownManager shutdownManager;
 
     protected ClusteredAgentManagerImpl() {
         super();
@@ -1341,8 +1348,10 @@ public String dispatch(final ClusterServicePdu pdu) {
                 return _gson.toJson(answers);
             } else if (cmds.length == 1 && cmds[0] instanceof ScheduleHostScanTaskCommand) {
                 final ScheduleHostScanTaskCommand cmd = (ScheduleHostScanTaskCommand)cmds[0];
-                final String response = handleScheduleHostScanTaskCommand(cmd);
-                return response;
+                return handleScheduleHostScanTaskCommand(cmd);
+            } else if (cmds.length == 1 && cmds[0] instanceof BaseShutdownManagementServerHostCommand) {
+                final BaseShutdownManagementServerHostCommand cmd = (BaseShutdownManagementServerHostCommand)cmds[0];
+                return handleShutdownManagementServerHostCommand(cmd);
             }
 
             try {
@@ -1376,6 +1385,36 @@ public String dispatch(final ClusterServicePdu pdu) {
             return null;
         }
 
+        private String handleShutdownManagementServerHostCommand(BaseShutdownManagementServerHostCommand cmd) {
+            if (cmd instanceof PrepareForShutdownManagementServerHostCommand) {
+                s_logger.debug("Received BaseShutdownManagementServerHostCommand - preparing to shut down");
+                try {
+                    shutdownManager.prepareForShutdown();
+                    return "Successfully prepared for shutdown";
+                } catch(CloudRuntimeException e) {
+                    return e.getMessage();
+                }
+            }
+            if (cmd instanceof TriggerShutdownManagementServerHostCommand) {
+                s_logger.debug("Received TriggerShutdownManagementServerHostCommand - triggering a shut down");
+                try {
+                    shutdownManager.triggerShutdown();
+                    return "Successfully triggered shutdown";
+                } catch(CloudRuntimeException e) {
+                    return e.getMessage();
+                }
+            }
+            if (cmd instanceof CancelShutdownManagementServerHostCommand) {
+                s_logger.debug("Received CancelShutdownManagementServerHostCommand - cancelling shut down");
+                try {
+                    shutdownManager.cancelShutdown();
+                    return "Successfully prepared for shutdown";
+                } catch(CloudRuntimeException e) {
+                    return e.getMessage();
+                }
+            }
+            throw new CloudRuntimeException("Unknown BaseShutdownManagementServerHostCommand command received : " + cmd);
+        }
     }
 
     public boolean executeAgentUserRequest(final long agentId, final Event event) throws AgentUnavailableException {

engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql

@@ -1582,4 +1582,4 @@ UPDATE
 SET
   usage_type = 22
 WHERE
-  usage_type = 24 AND usage_display like '% io write';
+  usage_type = 24 AND usage_display like '% io write';

engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql

@@ -19,3 +19,115 @@
 -- Schema upgrade from 4.18.1.0 to 4.19.0.0
 --;
 
+ALTER TABLE `cloud`.`mshost` MODIFY COLUMN `state` varchar(25);
+
+DROP VIEW IF EXISTS `cloud`.`async_job_view`;
+CREATE VIEW `cloud`.`async_job_view` AS
+    select
+        account.id account_id,
+        account.uuid account_uuid,
+        account.account_name account_name,
+        account.type account_type,
+        domain.id domain_id,
+        domain.uuid domain_uuid,
+        domain.name domain_name,
+        domain.path domain_path,
+        user.id user_id,
+        user.uuid user_uuid,
+        async_job.id,
+        async_job.uuid,
+        async_job.job_cmd,
+        async_job.job_status,
+        async_job.job_process_status,
+        async_job.job_result_code,
+        async_job.job_result,
+        async_job.created,
+        async_job.removed,
+        async_job.instance_type,
+        async_job.instance_id,
+        async_job.job_executing_msid,
+        CASE
+            WHEN async_job.instance_type = 'Volume' THEN volumes.uuid
+            WHEN
+                async_job.instance_type = 'Template'
+                    or async_job.instance_type = 'Iso'
+            THEN
+                vm_template.uuid
+            WHEN
+                async_job.instance_type = 'VirtualMachine'
+                    or async_job.instance_type = 'ConsoleProxy'
+                    or async_job.instance_type = 'SystemVm'
+                    or async_job.instance_type = 'DomainRouter'
+            THEN
+                vm_instance.uuid
+            WHEN async_job.instance_type = 'Snapshot' THEN snapshots.uuid
+            WHEN async_job.instance_type = 'Host' THEN host.uuid
+            WHEN async_job.instance_type = 'StoragePool' THEN storage_pool.uuid
+            WHEN async_job.instance_type = 'IpAddress' THEN user_ip_address.uuid
+            WHEN async_job.instance_type = 'SecurityGroup' THEN security_group.uuid
+            WHEN async_job.instance_type = 'PhysicalNetwork' THEN physical_network.uuid
+            WHEN async_job.instance_type = 'TrafficType' THEN physical_network_traffic_types.uuid
+            WHEN async_job.instance_type = 'PhysicalNetworkServiceProvider' THEN physical_network_service_providers.uuid
+            WHEN async_job.instance_type = 'FirewallRule' THEN firewall_rules.uuid
+            WHEN async_job.instance_type = 'Account' THEN acct.uuid
+            WHEN async_job.instance_type = 'User' THEN us.uuid
+            WHEN async_job.instance_type = 'StaticRoute' THEN static_routes.uuid
+            WHEN async_job.instance_type = 'PrivateGateway' THEN vpc_gateways.uuid
+            WHEN async_job.instance_type = 'Counter' THEN counter.uuid
+            WHEN async_job.instance_type = 'Condition' THEN conditions.uuid
+            WHEN async_job.instance_type = 'AutoScalePolicy' THEN autoscale_policies.uuid
+            WHEN async_job.instance_type = 'AutoScaleVmProfile' THEN autoscale_vmprofiles.uuid
+            WHEN async_job.instance_type = 'AutoScaleVmGroup' THEN autoscale_vmgroups.uuid
+            ELSE null
+        END instance_uuid
+    from
+        `cloud`.`async_job`
+            left join
+        `cloud`.`account` ON async_job.account_id = account.id
+            left join
+        `cloud`.`domain` ON domain.id = account.domain_id
+            left join
+        `cloud`.`user` ON async_job.user_id = user.id
+            left join
+        `cloud`.`volumes` ON async_job.instance_id = volumes.id
+            left join
+        `cloud`.`vm_template` ON async_job.instance_id = vm_template.id
+            left join
+        `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id
+            left join
+        `cloud`.`snapshots` ON async_job.instance_id = snapshots.id
+            left join
+        `cloud`.`host` ON async_job.instance_id = host.id
+            left join
+        `cloud`.`storage_pool` ON async_job.instance_id = storage_pool.id
+            left join
+        `cloud`.`user_ip_address` ON async_job.instance_id = user_ip_address.id
+            left join
+        `cloud`.`security_group` ON async_job.instance_id = security_group.id
+            left join
+        `cloud`.`physical_network` ON async_job.instance_id = physical_network.id
+            left join
+        `cloud`.`physical_network_traffic_types` ON async_job.instance_id = physical_network_traffic_types.id
+            left join
+        `cloud`.`physical_network_service_providers` ON async_job.instance_id = physical_network_service_providers.id
+            left join
+        `cloud`.`firewall_rules` ON async_job.instance_id = firewall_rules.id
+            left join
+        `cloud`.`account` acct ON async_job.instance_id = acct.id
+            left join
+        `cloud`.`user` us ON async_job.instance_id = us.id
+            left join
+        `cloud`.`static_routes` ON async_job.instance_id = static_routes.id
+            left join
+        `cloud`.`vpc_gateways` ON async_job.instance_id = vpc_gateways.id
+            left join
+        `cloud`.`counter` ON async_job.instance_id = counter.id
+            left join
+        `cloud`.`conditions` ON async_job.instance_id = conditions.id
+            left join
+        `cloud`.`autoscale_policies` ON async_job.instance_id = autoscale_policies.id
+            left join
+        `cloud`.`autoscale_vmprofiles` ON async_job.instance_id = autoscale_vmprofiles.id
+            left join
+        `cloud`.`autoscale_vmgroups` ON async_job.instance_id = autoscale_vmgroups.id;
+

framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJob.java

@@ -90,6 +90,8 @@ public static interface Constants {
     @Override
     Long getExecutingMsid();
 
+    void setExecutingMsid(Long msid);
+
     @Override
     Long getCompleteMsid();
 

framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java

@@ -133,4 +133,14 @@ void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakupDispa
     List<AsyncJobVO> findFailureAsyncJobs(String... cmds);
 
     long countPendingJobs(String havingInfo, String... cmds);
+
+    // Returns the number of pending jobs for the given Management server msids.
+    // NOTE: This is the msid and NOT the id
+    long countPendingNonPseudoJobs(Long... msIds);
+
+    void enableAsyncJobs();
+
+    void disableAsyncJobs();
+
+    boolean isAsyncJobsEnabled();
 }

framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java

@@ -46,4 +46,8 @@ public interface AsyncJobDao extends GenericDao<AsyncJobVO, Long> {
     List<AsyncJobVO> getFailureJobsSinceLastMsStart(long msId, String... cmds);
 
     long countPendingJobs(String havingInfo, String... cmds);
+
+    // Returns the number of pending jobs for the given Management server msids.
+    // NOTE: This is the msid and NOT the id
+    long countPendingNonPseudoJobs(Long... msIds);
 }