diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 6e93d45abdb2..c1438251b858 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -523,6 +523,7 @@ public class ApiConstants { public static final String PRIVATE_NETWORK_ID = "privatenetworkid"; public static final String ALLOCATION_STATE = "allocationstate"; public static final String MANAGED_STATE = "managedstate"; + public static final String MANAGEMENT_SERVER_ID = "managementserverid"; public static final String STORAGE_ID = "storageid"; public static final String PING_STORAGE_SERVER_IP = "pingstorageserverip"; public static final String PING_DIR = "pingdir"; @@ -1018,6 +1019,10 @@ public class ApiConstants { public static final String LOGOUT = "logout"; public static final String LIST_IDPS = "listIdps"; + public static final String READY_FOR_SHUTDOWN = "readyforshutdown"; + public static final String SHUTDOWN_TRIGGERED = "shutdowntriggered"; + public static final String PENDING_JOBS_COUNT = "pendingjobscount"; + public static final String PUBLIC_MTU = "publicmtu"; public static final String PRIVATE_MTU = "privatemtu"; public static final String MTU = "mtu"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index d2574ff442e0..783d78fdce32 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; @APICommand(name = "listAsyncJobs", description = "Lists all pending asynchronous jobs for the account.", responseObject = AsyncJobResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -36,6 +37,9 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date of the async job (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")") private Date startDate; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "The id of the management server", since="4.19") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -44,6 +48,10 @@ public Date getStartDate() { return startDate; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java index eecd6be6c521..3eeaaef2afac 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java @@ -32,9 +32,21 @@ public class AsyncJobResponse extends BaseResponse { @SerializedName("accountid") - @Param(description = "the account that executed the async command") + @Param(description = "the account id that executed the async command") private String accountId; + @SerializedName("account") + @Param(description = "the account that executed the async command") + private String account; + + @SerializedName("domainid") + @Param(description = "the domain id that executed the async command") + private String domainid; + + @SerializedName("domainpath") + @Param(description = "the domain that executed the async command") + private String domainPath; + @SerializedName(ApiConstants.USER_ID) @Param(description = "the user that executed the async command") private String userId; @@ -71,6 +83,10 @@ public class AsyncJobResponse extends BaseResponse { @Param(description = "the unique ID of the instance/entity object related to the job") private String jobInstanceId; + @SerializedName("managementserverid") + @Param(description = "the msid of the management server on which the job is running", since = "4.19") + private Long msid; + @SerializedName(ApiConstants.CREATED) @Param(description = " the created date of the job") private Date created; @@ -83,6 +99,18 @@ public void setAccountId(String accountId) { this.accountId = accountId; } + public void setAccount(String account) { + this.account = account; + } + + public void setDomainId(String domainid) { + this.domainid = domainid; + } + + public void setDomainPath(String domainPath) { + this.domainPath = domainPath; + } + public void setUserId(String userId) { this.userId = userId; } @@ -127,4 +155,8 @@ public void setCreated(Date created) { public void setRemoved(final Date removed) { this.removed = removed; } + + public void setMsid(Long msid) { + this.msid = msid; + } } diff --git a/api/src/main/java/org/apache/cloudstack/management/ManagementServerHost.java b/api/src/main/java/org/apache/cloudstack/management/ManagementServerHost.java index 0159fb2e7919..834291ef21c8 100644 --- a/api/src/main/java/org/apache/cloudstack/management/ManagementServerHost.java +++ b/api/src/main/java/org/apache/cloudstack/management/ManagementServerHost.java @@ -21,7 +21,7 @@ public interface ManagementServerHost extends InternalIdentity, Identity { enum State { - Up, Down + Up, Down, PreparingToShutDown, ReadyToShutDown, ShuttingDown } long getMsid(); diff --git a/client/pom.xml b/client/pom.xml index 0ff5c3203b93..a548d676a671 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -552,6 +552,11 @@ cloud-plugin-integrations-kubernetes-service ${project.version} + + org.apache.cloudstack + cloud-plugin-shutdown + ${project.version} + diff --git a/engine/orchestration/pom.xml b/engine/orchestration/pom.xml index 2cd17519e0ef..e5eeb88f6c5b 100755 --- a/engine/orchestration/pom.xml +++ b/engine/orchestration/pom.xml @@ -68,6 +68,11 @@ cloud-server ${project.version} + + org.apache.cloudstack + cloud-plugin-shutdown + ${project.version} + diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 749a738e63ed..bd4e259a7885 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -50,6 +50,11 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; +import org.apache.cloudstack.shutdown.ShutdownManager; +import org.apache.cloudstack.shutdown.command.CancelShutdownManagementServerHostCommand; +import org.apache.cloudstack.shutdown.command.PrepareForShutdownManagementServerHostCommand; +import org.apache.cloudstack.shutdown.command.BaseShutdownManagementServerHostCommand; +import org.apache.cloudstack.shutdown.command.TriggerShutdownManagementServerHostCommand; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.log4j.Logger; @@ -129,6 +134,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust private HAConfigDao haConfigDao; @Inject private CAManager caService; + @Inject + private ShutdownManager shutdownManager; protected ClusteredAgentManagerImpl() { super(); @@ -1341,8 +1348,10 @@ public String dispatch(final ClusterServicePdu pdu) { return _gson.toJson(answers); } else if (cmds.length == 1 && cmds[0] instanceof ScheduleHostScanTaskCommand) { final ScheduleHostScanTaskCommand cmd = (ScheduleHostScanTaskCommand)cmds[0]; - final String response = handleScheduleHostScanTaskCommand(cmd); - return response; + return handleScheduleHostScanTaskCommand(cmd); + } else if (cmds.length == 1 && cmds[0] instanceof BaseShutdownManagementServerHostCommand) { + final BaseShutdownManagementServerHostCommand cmd = (BaseShutdownManagementServerHostCommand)cmds[0]; + return handleShutdownManagementServerHostCommand(cmd); } try { @@ -1376,6 +1385,36 @@ public String dispatch(final ClusterServicePdu pdu) { return null; } + private String handleShutdownManagementServerHostCommand(BaseShutdownManagementServerHostCommand cmd) { + if (cmd instanceof PrepareForShutdownManagementServerHostCommand) { + s_logger.debug("Received BaseShutdownManagementServerHostCommand - preparing to shut down"); + try { + shutdownManager.prepareForShutdown(); + return "Successfully prepared for shutdown"; + } catch(CloudRuntimeException e) { + return e.getMessage(); + } + } + if (cmd instanceof TriggerShutdownManagementServerHostCommand) { + s_logger.debug("Received TriggerShutdownManagementServerHostCommand - triggering a shut down"); + try { + shutdownManager.triggerShutdown(); + return "Successfully triggered shutdown"; + } catch(CloudRuntimeException e) { + return e.getMessage(); + } + } + if (cmd instanceof CancelShutdownManagementServerHostCommand) { + s_logger.debug("Received CancelShutdownManagementServerHostCommand - cancelling shut down"); + try { + shutdownManager.cancelShutdown(); + return "Successfully prepared for shutdown"; + } catch(CloudRuntimeException e) { + return e.getMessage(); + } + } + throw new CloudRuntimeException("Unknown BaseShutdownManagementServerHostCommand command received : " + cmd); + } } public boolean executeAgentUserRequest(final long agentId, final Event event) throws AgentUnavailableException { diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql index b8eee33cad74..f5efb07dd779 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql @@ -1582,4 +1582,4 @@ UPDATE SET usage_type = 22 WHERE - usage_type = 24 AND usage_display like '% io write'; \ No newline at end of file + usage_type = 24 AND usage_display like '% io write'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 5e13b1c0157e..541d3dbacac3 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -19,3 +19,115 @@ -- Schema upgrade from 4.18.1.0 to 4.19.0.0 --; +ALTER TABLE `cloud`.`mshost` MODIFY COLUMN `state` varchar(25); + +DROP VIEW IF EXISTS `cloud`.`async_job_view`; +CREATE VIEW `cloud`.`async_job_view` AS + select + account.id account_id, + account.uuid account_uuid, + account.account_name account_name, + account.type account_type, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + user.id user_id, + user.uuid user_uuid, + async_job.id, + async_job.uuid, + async_job.job_cmd, + async_job.job_status, + async_job.job_process_status, + async_job.job_result_code, + async_job.job_result, + async_job.created, + async_job.removed, + async_job.instance_type, + async_job.instance_id, + async_job.job_executing_msid, + CASE + WHEN async_job.instance_type = 'Volume' THEN volumes.uuid + WHEN + async_job.instance_type = 'Template' + or async_job.instance_type = 'Iso' + THEN + vm_template.uuid + WHEN + async_job.instance_type = 'VirtualMachine' + or async_job.instance_type = 'ConsoleProxy' + or async_job.instance_type = 'SystemVm' + or async_job.instance_type = 'DomainRouter' + THEN + vm_instance.uuid + WHEN async_job.instance_type = 'Snapshot' THEN snapshots.uuid + WHEN async_job.instance_type = 'Host' THEN host.uuid + WHEN async_job.instance_type = 'StoragePool' THEN storage_pool.uuid + WHEN async_job.instance_type = 'IpAddress' THEN user_ip_address.uuid + WHEN async_job.instance_type = 'SecurityGroup' THEN security_group.uuid + WHEN async_job.instance_type = 'PhysicalNetwork' THEN physical_network.uuid + WHEN async_job.instance_type = 'TrafficType' THEN physical_network_traffic_types.uuid + WHEN async_job.instance_type = 'PhysicalNetworkServiceProvider' THEN physical_network_service_providers.uuid + WHEN async_job.instance_type = 'FirewallRule' THEN firewall_rules.uuid + WHEN async_job.instance_type = 'Account' THEN acct.uuid + WHEN async_job.instance_type = 'User' THEN us.uuid + WHEN async_job.instance_type = 'StaticRoute' THEN static_routes.uuid + WHEN async_job.instance_type = 'PrivateGateway' THEN vpc_gateways.uuid + WHEN async_job.instance_type = 'Counter' THEN counter.uuid + WHEN async_job.instance_type = 'Condition' THEN conditions.uuid + WHEN async_job.instance_type = 'AutoScalePolicy' THEN autoscale_policies.uuid + WHEN async_job.instance_type = 'AutoScaleVmProfile' THEN autoscale_vmprofiles.uuid + WHEN async_job.instance_type = 'AutoScaleVmGroup' THEN autoscale_vmgroups.uuid + ELSE null + END instance_uuid + from + `cloud`.`async_job` + left join + `cloud`.`account` ON async_job.account_id = account.id + left join + `cloud`.`domain` ON domain.id = account.domain_id + left join + `cloud`.`user` ON async_job.user_id = user.id + left join + `cloud`.`volumes` ON async_job.instance_id = volumes.id + left join + `cloud`.`vm_template` ON async_job.instance_id = vm_template.id + left join + `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id + left join + `cloud`.`snapshots` ON async_job.instance_id = snapshots.id + left join + `cloud`.`host` ON async_job.instance_id = host.id + left join + `cloud`.`storage_pool` ON async_job.instance_id = storage_pool.id + left join + `cloud`.`user_ip_address` ON async_job.instance_id = user_ip_address.id + left join + `cloud`.`security_group` ON async_job.instance_id = security_group.id + left join + `cloud`.`physical_network` ON async_job.instance_id = physical_network.id + left join + `cloud`.`physical_network_traffic_types` ON async_job.instance_id = physical_network_traffic_types.id + left join + `cloud`.`physical_network_service_providers` ON async_job.instance_id = physical_network_service_providers.id + left join + `cloud`.`firewall_rules` ON async_job.instance_id = firewall_rules.id + left join + `cloud`.`account` acct ON async_job.instance_id = acct.id + left join + `cloud`.`user` us ON async_job.instance_id = us.id + left join + `cloud`.`static_routes` ON async_job.instance_id = static_routes.id + left join + `cloud`.`vpc_gateways` ON async_job.instance_id = vpc_gateways.id + left join + `cloud`.`counter` ON async_job.instance_id = counter.id + left join + `cloud`.`conditions` ON async_job.instance_id = conditions.id + left join + `cloud`.`autoscale_policies` ON async_job.instance_id = autoscale_policies.id + left join + `cloud`.`autoscale_vmprofiles` ON async_job.instance_id = autoscale_vmprofiles.id + left join + `cloud`.`autoscale_vmgroups` ON async_job.instance_id = autoscale_vmgroups.id; + diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJob.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJob.java index b8200bf82219..bde9b4af1671 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJob.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJob.java @@ -90,6 +90,8 @@ public static interface Constants { @Override Long getExecutingMsid(); + void setExecutingMsid(Long msid); + @Override Long getCompleteMsid(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java index 8542407524b1..52ef10a4adc4 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java @@ -133,4 +133,14 @@ void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakupDispa List findFailureAsyncJobs(String... cmds); long countPendingJobs(String havingInfo, String... cmds); + + // Returns the number of pending jobs for the given Management server msids. + // NOTE: This is the msid and NOT the id + long countPendingNonPseudoJobs(Long... msIds); + + void enableAsyncJobs(); + + void disableAsyncJobs(); + + boolean isAsyncJobsEnabled(); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 2696e105cce4..9f7a4ad6e058 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -46,4 +46,8 @@ public interface AsyncJobDao extends GenericDao { List getFailureJobsSinceLastMsStart(long msId, String... cmds); long countPendingJobs(String havingInfo, String... cmds); + + // Returns the number of pending jobs for the given Management server msids. + // NOTE: This is the msid and NOT the id + long countPendingNonPseudoJobs(Long... msIds); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 7dd73435093d..1914ff714602 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -48,6 +48,7 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements private final SearchBuilder expiringCompletedAsyncJobSearch; private final SearchBuilder failureMsidAsyncJobSearch; private final GenericSearchBuilder asyncJobTypeSearch; + private final GenericSearchBuilder pendingNonPseudoAsyncJobsSearch; public AsyncJobDaoImpl() { pendingAsyncJobSearch = createSearchBuilder(); @@ -103,6 +104,11 @@ public AsyncJobDaoImpl() { asyncJobTypeSearch.and("status", asyncJobTypeSearch.entity().getStatus(), SearchCriteria.Op.EQ); asyncJobTypeSearch.done(); + pendingNonPseudoAsyncJobsSearch = createSearchBuilder(Long.class); + pendingNonPseudoAsyncJobsSearch.select(null, SearchCriteria.Func.COUNT, pendingNonPseudoAsyncJobsSearch.entity().getId()); + pendingNonPseudoAsyncJobsSearch.and("instanceTypeNEQ", pendingNonPseudoAsyncJobsSearch.entity().getInstanceType(), SearchCriteria.Op.NEQ); + pendingNonPseudoAsyncJobsSearch.and("jobStatusEQ", pendingNonPseudoAsyncJobsSearch.entity().getStatus(), SearchCriteria.Op.EQ); + pendingNonPseudoAsyncJobsSearch.and("executingMsidIN", pendingNonPseudoAsyncJobsSearch.entity().getExecutingMsid(), SearchCriteria.Op.IN); } @Override @@ -237,6 +243,20 @@ public List getFailureJobsSinceLastMsStart(long msId, String... cmds return listBy(sc); } + // Returns the number of pending jobs for the given Management server msids. + // NOTE: This is the msid and NOT the id + @Override + public long countPendingNonPseudoJobs(Long... msIds) { + SearchCriteria sc = pendingNonPseudoAsyncJobsSearch.create(); + sc.setParameters("instanceTypeNEQ", AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE); + sc.setParameters("jobStatusEQ", JobInfo.Status.IN_PROGRESS); + if (msIds != null) { + sc.setParameters("executingMsidIN", (Object[])msIds); + } + List results = customSearch(sc, null); + return results.get(0); + } + @Override public long countPendingJobs(String havingInfo, String... cmds) { SearchCriteria sc = asyncJobTypeSearch.create(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index a963357122e5..0546d998913a 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -64,6 +64,7 @@ import com.cloud.cluster.ClusterManagerListener; import org.apache.cloudstack.management.ManagementServerHost; + import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.dao.SnapshotDao; @@ -155,6 +156,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private ExecutorService _apiJobExecutor; private ExecutorService _workerJobExecutor; + private boolean asyncJobsEnabled = true; + @Override public String getConfigComponentName() { return AsyncJobManager.class.getSimpleName(); @@ -197,12 +200,21 @@ public long submitAsyncJob(AsyncJob job) { return submitAsyncJob(job, false); } + private void checkShutdown() { + if (!isAsyncJobsEnabled()) { + throw new CloudRuntimeException("A shutdown has been triggered. Can not accept new jobs"); + } + } + @SuppressWarnings("unchecked") @DB public long submitAsyncJob(AsyncJob job, boolean scheduleJobExecutionInContext) { + checkShutdown(); + @SuppressWarnings("rawtypes") GenericDao dao = GenericDaoBase.getDao(job.getClass()); job.setInitMsid(getMsid()); + job.setExecutingMsid(getMsid()); job.setSyncSource(null); // no sync source originally dao.persist(job); @@ -218,6 +230,8 @@ public long submitAsyncJob(AsyncJob job, boolean scheduleJobExecutionInContext) @Override @DB public long submitAsyncJob(final AsyncJob job, final String syncObjType, final long syncObjId) { + checkShutdown(); + try { @SuppressWarnings("rawtypes") final GenericDao dao = GenericDaoBase.getDao(job.getClass()); @@ -827,6 +841,11 @@ protected void runInContext() { protected void reallyRun() { try { + if (!isAsyncJobsEnabled()) { + s_logger.info("A shutdown has been triggered. Not executing any async job"); + return; + } + List l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE); if (l != null && l.size() > 0) { for (SyncQueueItemVO item : l) { @@ -1171,4 +1190,26 @@ public List findFailureAsyncJobs(String... cmds) { public long countPendingJobs(String havingInfo, String... cmds) { return _jobDao.countPendingJobs(havingInfo, cmds); } + + // Returns the number of pending jobs for the given Management server msids. + // NOTE: This is the msid and NOT the id + @Override + public long countPendingNonPseudoJobs(Long... msIds) { + return _jobDao.countPendingNonPseudoJobs(msIds); + } + + @Override + public void enableAsyncJobs() { + this.asyncJobsEnabled = true; + } + + @Override + public void disableAsyncJobs() { + this.asyncJobsEnabled = false; + } + + @Override + public boolean isAsyncJobsEnabled() { + return asyncJobsEnabled; + } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java index 8f3c0337837b..6b85ae27f58a 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java @@ -294,6 +294,7 @@ public Long getExecutingMsid() { return executingMsid; } + @Override public void setExecutingMsid(Long executingMsid) { this.executingMsid = executingMsid; } diff --git a/plugins/pom.xml b/plugins/pom.xml index 529cf3832887..d0661c01a2c4 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -115,6 +115,8 @@ outofbandmanagement-drivers/nested-cloudstack outofbandmanagement-drivers/redfish + shutdown + storage/image/default storage/image/s3 storage/image/sample diff --git a/plugins/shutdown/pom.xml b/plugins/shutdown/pom.xml new file mode 100644 index 000000000000..0f34cc084828 --- /dev/null +++ b/plugins/shutdown/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + cloud-plugin-shutdown + Apache CloudStack Plugin - Safe Shutdown + + org.apache.cloudstack + cloudstack-plugins + 4.19.0.0-SNAPSHOT + ../pom.xml + + + + org.apache.cloudstack + cloud-api + ${project.version} + + + org.apache.cloudstack + cloud-utils + ${project.version} + + + diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/BaseShutdownActionCmd.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/BaseShutdownActionCmd.java new file mode 100644 index 000000000000..d7f4953291b6 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/BaseShutdownActionCmd.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; + + +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.shutdown.ShutdownManager; + +public abstract class BaseShutdownActionCmd extends BaseCmd { + + @Inject + protected ShutdownManager shutdownManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "the uuid of the management server", required = true) + private Long managementServerId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getManagementServerId() { + return managementServerId; + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/CancelShutdownCmd.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/CancelShutdownCmd.java new file mode 100644 index 000000000000..fe6204fd0cc3 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/CancelShutdownCmd.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + package org.apache.cloudstack.api.command; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.log4j.Logger; + +import com.cloud.user.Account; + +import org.apache.cloudstack.api.response.ReadyForShutdownResponse; +import org.apache.cloudstack.acl.RoleType; + +@APICommand(name = CancelShutdownCmd.APINAME, + description = "Cancels a triggered shutdown", + since = "4.19.0", + responseObject = ReadyForShutdownResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) + +public class CancelShutdownCmd extends BaseShutdownActionCmd { + + public static final Logger LOG = Logger.getLogger(CancelShutdownCmd.class); + public static final String APINAME = "cancelShutdown"; + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + final ReadyForShutdownResponse response = shutdownManager.cancelShutdown(this); + response.setResponseName(getCommandName()); + response.setObjectName("cancelshutdown"); + setResponseObject(response); + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/PrepareForShutdownCmd.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/PrepareForShutdownCmd.java new file mode 100644 index 000000000000..01ea1797a105 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/PrepareForShutdownCmd.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command; + + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.log4j.Logger; + +import com.cloud.user.Account; + +import org.apache.cloudstack.api.response.ReadyForShutdownResponse; +import org.apache.cloudstack.acl.RoleType; + +@APICommand(name = PrepareForShutdownCmd.APINAME, + description = "Prepares CloudStack for a safe manual shutdown by preventing new jobs from being accepted", + since = "4.19.0", + responseObject = ReadyForShutdownResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class PrepareForShutdownCmd extends BaseShutdownActionCmd { + public static final Logger LOG = Logger.getLogger(PrepareForShutdownCmd.class); + public static final String APINAME = "prepareForShutdown"; + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + final ReadyForShutdownResponse response = shutdownManager.prepareForShutdown(this); + response.setResponseName(getCommandName()); + response.setObjectName("prepareforshutdown"); + setResponseObject(response); + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/ReadyForShutdownCmd.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/ReadyForShutdownCmd.java new file mode 100644 index 000000000000..d7ab6a24ee68 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/ReadyForShutdownCmd.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.ReadyForShutdownResponse; +import org.apache.cloudstack.shutdown.ShutdownManager; +import org.apache.log4j.Logger; +import com.cloud.user.Account; + +@APICommand(name = ReadyForShutdownCmd.APINAME, + description = "Returs the status of CloudStack, whether a shutdown has been triggered and if ready to shutdown", + since = "4.19.0", + responseObject = ReadyForShutdownResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ReadyForShutdownCmd extends BaseCmd { + public static final Logger LOG = Logger.getLogger(ReadyForShutdownCmd.class); + public static final String APINAME = "readyForShutdown"; + + @Inject + private ShutdownManager shutdownManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "the uuid of the management server") + private Long managementServerId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getManagementServerId() { + return managementServerId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + final ReadyForShutdownResponse response = shutdownManager.readyForShutdown(this); + response.setResponseName(getCommandName()); + response.setObjectName("readyforshutdown"); + setResponseObject(response); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/TriggerShutdownCmd.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/TriggerShutdownCmd.java new file mode 100644 index 000000000000..3abde0b1f3b1 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/command/TriggerShutdownCmd.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.log4j.Logger; + +import com.cloud.user.Account; + +import org.apache.cloudstack.api.response.ReadyForShutdownResponse; +import org.apache.cloudstack.acl.RoleType; + +@APICommand(name = TriggerShutdownCmd.APINAME, + description = "Triggers an automatic safe shutdown of CloudStack by not accepting new jobs and shutting down when all pending jobbs have been completed. Triggers an immediate shutdown if forced", + since = "4.19.0", + responseObject = ReadyForShutdownResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class TriggerShutdownCmd extends BaseShutdownActionCmd { + public static final Logger LOG = Logger.getLogger(TriggerShutdownCmd.class); + public static final String APINAME = "triggerShutdown"; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + final ReadyForShutdownResponse response = shutdownManager.triggerShutdown(this); + response.setResponseName(getCommandName()); + response.setObjectName("triggershutdown"); + setResponseObject(response); + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/api/response/ReadyForShutdownResponse.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/response/ReadyForShutdownResponse.java new file mode 100644 index 000000000000..d1b2353d2a33 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/api/response/ReadyForShutdownResponse.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class ReadyForShutdownResponse extends BaseResponse { + @SerializedName(ApiConstants.READY_FOR_SHUTDOWN) + @Param(description = "Indicates whether CloudStack is ready to shutdown") + private Boolean readyForShutdown; + + @SerializedName(ApiConstants.SHUTDOWN_TRIGGERED) + @Param(description = "Indicates whether a shutdown has been triggered") + private Boolean shutdownTriggered; + + @SerializedName(ApiConstants.PENDING_JOBS_COUNT) + @Param(description = "The number of jobs in progress") + private Long pendingJobsCount; + + @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID) + @Param(description = "The id of the management server") + private Long msId; + + public ReadyForShutdownResponse(Long msId, Boolean shutdownTriggered, Boolean readyForShutdown, long pendingJobsCount) { + this.msId = msId; + this.shutdownTriggered = shutdownTriggered; + this.readyForShutdown = readyForShutdown; + this.pendingJobsCount = pendingJobsCount; + } + + public Boolean getShutdownTriggered() { + return this.shutdownTriggered; + } + + public void setShutdownTriggered(Boolean shutdownTriggered) { + this.shutdownTriggered = shutdownTriggered; + } + + public Boolean getReadyForShutdown() { + return this.readyForShutdown; + } + + public void setReadyForShutdown(Boolean readyForShutdown) { + this.readyForShutdown = readyForShutdown; + } + + public Long getPendingJobsCount() { + return this.pendingJobsCount; + } + + public void setPendingJobsCount(Long pendingJobsCount) { + this.pendingJobsCount = pendingJobsCount; + } + + public Long getMsId() { + return msId; + } + + public void setMsId(Long msId) { + this.msId = msId; + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/ShutdownManager.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/ShutdownManager.java new file mode 100644 index 000000000000..22f43cb4f626 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/ShutdownManager.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.shutdown; + +import org.apache.cloudstack.api.command.CancelShutdownCmd; +import org.apache.cloudstack.api.command.PrepareForShutdownCmd; +import org.apache.cloudstack.api.command.ReadyForShutdownCmd; +import org.apache.cloudstack.api.command.TriggerShutdownCmd; +import org.apache.cloudstack.api.response.ReadyForShutdownResponse; + +public interface ShutdownManager { + // Returns the number of pending jobs for the given Management server msids. + // NOTE: This is the msid and NOT the id + long countPendingJobs(Long... msIds); + + // Indicates whether a shutdown has been triggered on the current management server + boolean isShutdownTriggered(); + + // Indicates whether the current management server is preparing to shutdown + boolean isPreparingForShutdown(); + + // Triggers a shutdown on the current management server by not accepting any more async jobs and shutting down when there are no pending jobs + void triggerShutdown(); + + // Prepares the current management server to shutdown by not accepting any more async jobs + void prepareForShutdown(); + + // Cancels the shutdown on the current management server + void cancelShutdown(); + + // Returns whether the given ms can be shut down + ReadyForShutdownResponse readyForShutdown(Long managementserverid); + + // Returns whether the any of the ms can be shut down and if a shutdown has been triggered on any running ms + ReadyForShutdownResponse readyForShutdown(ReadyForShutdownCmd cmd); + + // Prepares the specified management server to shutdown by not accepting any more async jobs + ReadyForShutdownResponse prepareForShutdown(PrepareForShutdownCmd cmd); + + // Cancels the shutdown on the specified management server + ReadyForShutdownResponse cancelShutdown(CancelShutdownCmd cmd); + + // Triggers a shutdown on the specified management server by not accepting any more async jobs and shutting down when there are no pending jobs + ReadyForShutdownResponse triggerShutdown(TriggerShutdownCmd cmd); +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/ShutdownManagerImpl.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/ShutdownManagerImpl.java new file mode 100644 index 000000000000..b8f5fb57155d --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/ShutdownManagerImpl.java @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.shutdown; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.command.CancelShutdownCmd; +import org.apache.cloudstack.api.command.PrepareForShutdownCmd; +import org.apache.cloudstack.api.command.ReadyForShutdownCmd; +import org.apache.cloudstack.api.command.TriggerShutdownCmd; +import org.apache.cloudstack.api.response.ReadyForShutdownResponse; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.management.ManagementServerHost.State; +import org.apache.cloudstack.shutdown.command.CancelShutdownManagementServerHostCommand; +import org.apache.cloudstack.shutdown.command.PrepareForShutdownManagementServerHostCommand; +import org.apache.cloudstack.shutdown.command.TriggerShutdownManagementServerHostCommand; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; + +import com.cloud.agent.api.Command; +import com.cloud.cluster.ClusterManager; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.component.PluggableService; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.gson.Gson; + +public class ShutdownManagerImpl extends ManagerBase implements ShutdownManager, PluggableService{ + + private static Logger logger = Logger.getLogger(ShutdownManagerImpl.class); + Gson gson; + + @Inject + private AsyncJobManager jobManager; + @Inject + private ManagementServerHostDao msHostDao; + @Inject + private ClusterManager clusterManager; + + private boolean shutdownTriggered = false; + private boolean preparingForShutdown = false; + + private Timer timer = new Timer(); + private TimerTask shutdownTask; + + protected ShutdownManagerImpl() { + super(); + gson = GsonHelper.getGson(); + } + + @Override + public boolean isShutdownTriggered() { + return shutdownTriggered; + } + + @Override + public boolean isPreparingForShutdown() { + return preparingForShutdown; + } + + @Override + public long countPendingJobs(Long... msIds) { + return jobManager.countPendingNonPseudoJobs(msIds); + } + + @Override + public void triggerShutdown() { + if (this.shutdownTriggered) { + throw new CloudRuntimeException("A shutdown has already been triggered"); + } + this.shutdownTriggered = true; + prepareForShutdown(true); + } + + private void prepareForShutdown(boolean postTrigger) { + // Ensure we don't throw an error if triggering a shutdown after just preparing for it + if (!postTrigger && this.preparingForShutdown) { + throw new CloudRuntimeException("A shutdown has already been triggered"); + } + this.preparingForShutdown = true; + jobManager.disableAsyncJobs(); + if (this.shutdownTask != null) { + this.shutdownTask.cancel(); + this.shutdownTask = null; + } + this.shutdownTask = new ShutdownTask(this); + timer.scheduleAtFixedRate(shutdownTask, 0, 30L * 1000); + } + + @Override + public void prepareForShutdown() { + prepareForShutdown(false); + } + + @Override + public void cancelShutdown() { + if (!this.preparingForShutdown) { + throw new CloudRuntimeException("A shutdown has not been triggered"); + } + + this.preparingForShutdown = false; + this.shutdownTriggered = false; + jobManager.enableAsyncJobs(); + if (shutdownTask != null) { + shutdownTask.cancel(); + } + shutdownTask = null; + } + + @Override + public ReadyForShutdownResponse readyForShutdown(Long managementserverid) { + Long[] msIds = null; + boolean shutdownTriggeredAnywhere = false; + State[] shutdownTriggeredStates = {State.ShuttingDown, State.PreparingToShutDown, State.ReadyToShutDown}; + if (managementserverid == null) { + List msHosts = msHostDao.listBy(shutdownTriggeredStates); + if (msHosts != null && !msHosts.isEmpty()) { + msIds = new Long[msHosts.size()]; + for (int i = 0; i < msHosts.size(); i++) { + msIds[i] = msHosts.get(i).getMsid(); + } + shutdownTriggeredAnywhere = !msHosts.isEmpty(); + } + } else { + ManagementServerHostVO msHost = msHostDao.findById(managementserverid); + msIds = new Long[]{msHost.getMsid()}; + shutdownTriggeredAnywhere = Arrays.asList(shutdownTriggeredStates).contains(msHost.getState()); + } + long pendingJobCount = countPendingJobs(msIds); + return new ReadyForShutdownResponse(managementserverid, shutdownTriggeredAnywhere, pendingJobCount == 0, pendingJobCount); + } + + @Override + public ReadyForShutdownResponse readyForShutdown(ReadyForShutdownCmd cmd) { + return readyForShutdown(cmd.getManagementServerId()); + } + + @Override + public ReadyForShutdownResponse prepareForShutdown(PrepareForShutdownCmd cmd) { + ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); + final Command[] cmds = new Command[1]; + cmds[0] = new PrepareForShutdownManagementServerHostCommand(msHost.getMsid()); + String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); + logger.info("PrepareForShutdownCmd result : " + result); + if (!result.contains("Success")) { + throw new CloudRuntimeException(result); + } + + msHost.setState(State.PreparingToShutDown); + msHostDao.persist(msHost); + + return readyForShutdown(cmd.getManagementServerId()); + } + + @Override + public ReadyForShutdownResponse triggerShutdown(TriggerShutdownCmd cmd) { + ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); + final Command[] cmds = new Command[1]; + cmds[0] = new TriggerShutdownManagementServerHostCommand(msHost.getMsid()); + String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); + logger.info("TriggerShutdownCmd result : " + result); + if (!result.contains("Success")) { + throw new CloudRuntimeException(result); + } + + msHost.setState(State.ShuttingDown); + msHostDao.persist(msHost); + + return readyForShutdown(cmd.getManagementServerId()); + } + + @Override + public ReadyForShutdownResponse cancelShutdown(CancelShutdownCmd cmd) { + ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); + final Command[] cmds = new Command[1]; + cmds[0] = new CancelShutdownManagementServerHostCommand(msHost.getMsid()); + String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); + logger.info("CancelShutdownCmd result : " + result); + if (!result.contains("Success")) { + throw new CloudRuntimeException(result); + } + + msHost.setState(State.Up); + msHostDao.persist(msHost); + + return readyForShutdown(cmd.getManagementServerId()); + } + + @Override + public List> getCommands() { + final List> cmdList = new ArrayList<>(); + cmdList.add(CancelShutdownCmd.class); + cmdList.add(PrepareForShutdownCmd.class); + cmdList.add(ReadyForShutdownCmd.class); + cmdList.add(TriggerShutdownCmd.class); + return cmdList; + } + + private final class ShutdownTask extends TimerTask { + + private ShutdownManager shutdownManager; + + public ShutdownTask(ShutdownManager shutdownManager) { + this.shutdownManager = shutdownManager; + } + + @Override + public void run() { + try { + Long totalPendingJobs = shutdownManager.countPendingJobs(ManagementServerNode.getManagementServerId()); + String msg = String.format("Checking for triggered shutdown... shutdownTriggered [%b] AllowAsyncJobs [%b] PendingJobCount [%d]", + shutdownManager.isShutdownTriggered(), shutdownManager.isPreparingForShutdown(), totalPendingJobs); + logger.info(msg); + + // If the shutdown has been cancelled + if (!shutdownManager.isPreparingForShutdown()) { + logger.info("Shutdown cancelled. Terminating the shutdown timer task"); + this.cancel(); + return; + } + + // No more pending jobs. Good to terminate + if (totalPendingJobs == 0) { + if (shutdownManager.isShutdownTriggered()) { + logger.info("Shutting down now"); + System.exit(0); + } + if (shutdownManager.isPreparingForShutdown()) { + logger.info("Ready to shutdown"); + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + msHost.setState(State.ReadyToShutDown); + msHostDao.persist(msHost); + } + } + + logger.info("Pending jobs. Trying again later"); + } catch (final Exception e) { + logger.error("Error trying to run shutdown task", e); + } + } + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/BaseShutdownManagementServerHostCommand.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/BaseShutdownManagementServerHostCommand.java new file mode 100644 index 000000000000..8fe33317bc0c --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/BaseShutdownManagementServerHostCommand.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.cloudstack.shutdown.command; + +import com.cloud.agent.api.Command; + +public class BaseShutdownManagementServerHostCommand extends Command { + long msId; + + public BaseShutdownManagementServerHostCommand(long msId) { + this.msId = msId; + } + + public long getMsId() { + return msId; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/CancelShutdownManagementServerHostCommand.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/CancelShutdownManagementServerHostCommand.java new file mode 100644 index 000000000000..eef44446aa14 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/CancelShutdownManagementServerHostCommand.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.cloudstack.shutdown.command; + +public class CancelShutdownManagementServerHostCommand extends BaseShutdownManagementServerHostCommand { + + public CancelShutdownManagementServerHostCommand(long msId) { + super(msId); + } + +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/PrepareForShutdownManagementServerHostCommand.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/PrepareForShutdownManagementServerHostCommand.java new file mode 100644 index 000000000000..32a9201d5516 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/PrepareForShutdownManagementServerHostCommand.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.cloudstack.shutdown.command; + +public class PrepareForShutdownManagementServerHostCommand extends BaseShutdownManagementServerHostCommand { + + public PrepareForShutdownManagementServerHostCommand(long msId) { + super(msId); + } +} diff --git a/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/TriggerShutdownManagementServerHostCommand.java b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/TriggerShutdownManagementServerHostCommand.java new file mode 100644 index 000000000000..e0d1879fa358 --- /dev/null +++ b/plugins/shutdown/src/main/java/org/apache/cloudstack/shutdown/command/TriggerShutdownManagementServerHostCommand.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.cloudstack.shutdown.command; + +public class TriggerShutdownManagementServerHostCommand extends BaseShutdownManagementServerHostCommand { + + public TriggerShutdownManagementServerHostCommand(long msId) { + super(msId); + } +} diff --git a/plugins/shutdown/src/main/resources/META-INF/cloudstack/shutdown/module.properties b/plugins/shutdown/src/main/resources/META-INF/cloudstack/shutdown/module.properties new file mode 100644 index 000000000000..fd85c3085ca1 --- /dev/null +++ b/plugins/shutdown/src/main/resources/META-INF/cloudstack/shutdown/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=shutdown +parent=api diff --git a/plugins/shutdown/src/main/resources/META-INF/cloudstack/shutdown/spring-shutdown-context.xml b/plugins/shutdown/src/main/resources/META-INF/cloudstack/shutdown/spring-shutdown-context.xml new file mode 100644 index 000000000000..5318b3bf4462 --- /dev/null +++ b/plugins/shutdown/src/main/resources/META-INF/cloudstack/shutdown/spring-shutdown-context.xml @@ -0,0 +1,29 @@ + + + + + + + + diff --git a/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java b/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java new file mode 100644 index 000000000000..19ded7844db3 --- /dev/null +++ b/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.shutdown; + +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.exception.CloudRuntimeException; + + +@RunWith(MockitoJUnitRunner.class) +public class ShutdownManagerImplTest { + + @Spy + @InjectMocks + ShutdownManagerImpl spy; + + @Mock + AsyncJobManager jobManagerMock; + + private long prepareCountPendingJobs() { + long expectedCount = 1L; + Mockito.doReturn(expectedCount).when(jobManagerMock).countPendingNonPseudoJobs(1L); + return expectedCount; + } + + @Test + public void countPendingJobs() { + long expectedCount = prepareCountPendingJobs(); + long count = spy.countPendingJobs(1L); + Assert.assertEquals(expectedCount, count); + } + + @Test + public void cancelShutdown() { + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.cancelShutdown(); + }); + } + + @Test + public void prepareForShutdown() { + Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); + spy.prepareForShutdown(); + Mockito.verify(jobManagerMock).disableAsyncJobs(); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForShutdown(); + }); + + + Mockito.doNothing().when(jobManagerMock).enableAsyncJobs(); + spy.cancelShutdown(); + Mockito.verify(jobManagerMock).enableAsyncJobs(); + } +} diff --git a/server/src/main/java/com/cloud/api/ApiDispatcher.java b/server/src/main/java/com/cloud/api/ApiDispatcher.java index 3880f2aa9d1b..09a7a92a4a16 100644 --- a/server/src/main/java/com/cloud/api/ApiDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiDispatcher.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; import org.apache.log4j.Logger; import com.cloud.api.dispatch.DispatchChain; @@ -44,6 +45,7 @@ import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.exception.CloudRuntimeException; public class ApiDispatcher { private static final Logger s_logger = Logger.getLogger(ApiDispatcher.class.getName()); @@ -63,6 +65,9 @@ public class ApiDispatcher { @Inject() protected DispatchChainFactory dispatchChainFactory; + @Inject + AsyncJobManagerImpl asyncJobManager; + protected DispatchChain standardDispatchChain; protected DispatchChain asyncCreationDispatchChain; @@ -85,7 +90,11 @@ public void setMigrateQueueSizeLimit(final Long migrateLimit) { } public void dispatchCreateCmd(final BaseAsyncCreateCmd cmd, final Map params) throws Exception { - asyncCreationDispatchChain.dispatch(new DispatchTask(cmd, params)); + if (asyncJobManager.isAsyncJobsEnabled()) { + asyncCreationDispatchChain.dispatch(new DispatchTask(cmd, params)); + } else { + throw new CloudRuntimeException("A shutdown has been triggered. Can not accept new jobs"); + } } private void doAccessChecks(BaseCmd cmd, Map entitiesToAccess) { diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index c33aac83b400..d6c166e7ed7f 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -184,6 +184,8 @@ import com.cloud.api.query.vo.UserAccountJoinVO; import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.api.query.vo.VolumeJoinVO; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.dc.DataCenter; import com.cloud.dc.DedicatedResourceVO; import com.cloud.dc.dao.DedicatedResourceDao; @@ -455,6 +457,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ResourceIconDao resourceIconDao; + @Inject + private ManagementServerHostDao msHostDao; + + @Inject EntityManager entityManager; @@ -2544,6 +2550,10 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync } } + if (cmd.getManagementServerId() != null) { + sb.and("executingMsid", sb.entity().getExecutingMsid(), SearchCriteria.Op.EQ); + } + Object keyword = cmd.getKeyword(); Object startDate = cmd.getStartDate(); @@ -2572,6 +2582,11 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync sc.addAnd("created", SearchCriteria.Op.GTEQ, startDate); } + if (cmd.getManagementServerId() != null) { + ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); + sc.setParameters("executingMsid", msHost.getMsid()); + } + return _jobJoinDao.searchAndCount(sc, searchFilter); } diff --git a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java index bd110154e37c..32cd1c21dd6c 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java @@ -53,6 +53,11 @@ protected AsyncJobJoinDaoImpl() { public AsyncJobResponse newAsyncJobResponse(final AsyncJobJoinVO job) { final AsyncJobResponse jobResponse = new AsyncJobResponse(); jobResponse.setAccountId(job.getAccountUuid()); + jobResponse.setAccount(job.getAccountName()); + jobResponse.setDomainId(job.getDomainUuid()); + StringBuilder domainPath = new StringBuilder("ROOT"); + (domainPath.append(job.getDomainPath())).deleteCharAt(domainPath.length() - 1); + jobResponse.setDomainPath(domainPath.toString()); jobResponse.setUserId(job.getUserUuid()); jobResponse.setCmd(job.getCmd()); jobResponse.setCreated(job.getCreated()); @@ -60,6 +65,7 @@ public AsyncJobResponse newAsyncJobResponse(final AsyncJobJoinVO job) { jobResponse.setJobId(job.getUuid()); jobResponse.setJobStatus(job.getStatus()); jobResponse.setJobProcStatus(job.getProcessStatus()); + jobResponse.setMsid(job.getExecutingMsid()); if (job.getInstanceType() != null && job.getInstanceId() != null) { jobResponse.setJobInstanceType(job.getInstanceType().toString()); diff --git a/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java index a4db864367ff..cee5526796b4 100644 --- a/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java @@ -75,6 +75,9 @@ public class AsyncJobJoinVO extends BaseViewVO implements ControlledViewEntity { @Column(name = "job_cmd") private String cmd; + @Column(name = "job_executing_msid") + private Long executingMsid; + @Column(name = "job_status") private int status; @@ -214,6 +217,10 @@ public String getName() { return null; } + public Long getExecutingMsid() { + return executingMsid; + } + @Override public String getProjectUuid() { // TODO Auto-generated method stub diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index a9db15979c98..8bebc59a5187 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -21,10 +21,10 @@ xmlns:context="http://www.springframework.org/schema/context" xmlns:aop="http://www.springframework.org/schema/aop" xmlns:util="http://www.springframework.org/schema/util" - + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd - http://www.springframework.org/schema/aop + http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd @@ -129,7 +129,7 @@ - + @@ -211,45 +211,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -257,6 +257,9 @@ + + + @@ -265,14 +268,14 @@ - + - + diff --git a/test/integration/smoke/test_safe_shutdown.py b/test/integration/smoke/test_safe_shutdown.py new file mode 100644 index 000000000000..8277ae923f96 --- /dev/null +++ b/test/integration/smoke/test_safe_shutdown.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +from marvin.cloudstackTestCase import * +from marvin.cloudstackAPI import * +from marvin.lib.utils import * +from marvin.lib.base import * +from marvin.lib.common import * + +class TestSafeShutdown(cloudstackTestCase): + """ + Tests safely shutting down the Management Server + """ + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__ + self.cleanup = [] + + def tearDown(self): + super(TestSafeShutdown, self).tearDown() + + def isServerShutdown(self): + sshClient = SshClient( + self.mgtSvrDetails["mgtSvrIp"], + 22, + self.mgtSvrDetails["user"], + self.mgtSvrDetails["passwd"] + ) + + timeout = time.time() + 300 + while time.time() < timeout: + command = "service cloudstack-management status | grep dead" + results = sshClient.execute(command) + + if len(results) > 0 and "(dead)" in results[0] : + return + time.sleep(30) + return self.fail("Management server did shut down, failing") + + def isManagementUp(self): + try: + self.apiclient.listInfrastructure(listInfrastructure.listInfrastructureCmd()) + return True + except Exception: + return False + + def startServer(self): + """Start management server""" + + sshClient = SshClient( + self.mgtSvrDetails["mgtSvrIp"], + 22, + self.mgtSvrDetails["user"], + self.mgtSvrDetails["passwd"] + ) + + command = "service cloudstack-management start" + sshClient.execute(command) + + #Waits for management to come up in 5 mins, when it's up it will continue + timeout = time.time() + 300 + while time.time() < timeout: + if self.isManagementUp() is True: return + time.sleep(5) + return self.fail("Management server did not come up, failing") + + def run_async_cmd(self) : + return Project.create( + self.apiclient, + {"name": "test", "displaytext": "test"} + ) + + @attr(tags=["advanced", "smoke"]) + def test_01_prepare_and_cancel_shutdown(self): + try : + prepare_for_shutdown_cmd = prepareForShutdown.prepareForShutdownCmd() + prepare_for_shutdown_cmd.managementserverid = 1 + self.apiclient.prepareForShutdown(prepare_for_shutdown_cmd) + try : + self.run_async_cmd() + except Exception as e: + self.debug("Prepare for shutdown check successful, API failure: %s" % e) + finally : + cancel_shutdown_cmd = cancelShutdown.cancelShutdownCmd() + cancel_shutdown_cmd.managementserverid = 1 + response = self.apiclient.cancelShutdown(cancel_shutdown_cmd) + self.assertEqual( + response.shutdowntriggered, + False, + "Failed to cancel shutdown" + ) + ## Just to be sure, run another async command + project = self.run_async_cmd() + self.cleanup.append(project) + + @attr(tags=["advanced", "smoke"]) + def test_02_trigger_shutdown(self): + try : + cmd = triggerShutdown.triggerShutdownCmd() + cmd.managementserverid = 1 + self.apiclient.triggerShutdown(cmd) + self.isServerShutdown() + finally : + self.startServer() diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 1352800c2bf7..bd102e9c7cc1 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -248,7 +248,8 @@ 'Rolling': 'Rolling Maintenance', 'importVsphereStoragePolicies' : 'vSphere storage policies', 'listVsphereStoragePolicies' : 'vSphere storage policies', - 'ConsoleEndpoint': 'Console Endpoint' + 'ConsoleEndpoint': 'Console Endpoint', + 'Shutdown': 'Shutdown' } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index cdc013f82dcc..d66f936f2c0f 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -344,7 +344,6 @@ "label.asyncbackup": "Async backup", "label.authentication.method": "Authentication Method", "label.authentication.sshkey": "System SSH Key", -"label.automigrate.volume": "Auto migrate volume to another storage pool if required", "label.autoscale": "AutoScale", "label.autoscalevmgroupname": "AutoScale VM Group", "label.author.email": "Author e-mail", @@ -404,6 +403,7 @@ "label.bypassvlanoverlapcheck": "Bypass VLAN id/range overlap", "label.cachemode": "Write-cache type", "label.cancel": "Cancel", +"label.cancel.shutdown": "Cancel Shutdown", "label.capacity": "Capacity", "label.capacitybytes": "Capacity bytes", "label.capacityiops": "IOPS total", @@ -453,6 +453,7 @@ "label.collectiontime": "Collection time", "label.columns": "Columns", "label.comma.separated.list.description": "Enter comma-separated list of commands", +"label.command": "Command", "label.comments": "Comments", "label.communities": "Communities", "label.community": "Community", @@ -1408,6 +1409,7 @@ "label.pavr": "Virtual router", "label.payload": "Payload", "label.pcidevice": "GPU", +"label.pending.jobs": "Pending Jobs", "label.per.account": "Per account", "label.per.zone": "Per zone", "label.percentage": "Percentage", @@ -1446,6 +1448,7 @@ "label.preferred": "Preferred", "label.prefix": "Prefix", "label.prefix.type": "Prefix type", +"label.prepare.for.shutdown": "Prepare for Shutdown", "label.presetup": "PreSetup", "label.prev": "Prev", "label.previous": "Previous", @@ -1958,6 +1961,7 @@ "label.traffic.types": "Traffic types", "label.traffictype": "Traffic type", "label.transportzoneuuid": "Transport zone UUID", +"label.trigger.shutdown": "Trigger Safe Shutdown", "label.try.again": "Try again", "label.tuesday": "Tuesday", "label.two.factor.authentication.secret.key": "Your Two factor authentication secret key", @@ -2337,6 +2341,7 @@ "message.backup.create": "Are you sure you want create a VM backup?", "message.backup.offering.remove": "Are you sure you want to remove VM from backup offering and delete the backup chain?", "message.backup.restore": "Please confirm that you want to restore the vm backup?", +"message.cancel.shutdown": "Please confirm that you would like to cancel the shutdown on this Management server. It will resume accepting any new Async Jobs.", "message.certificate.upload.processing": "Certificate upload in progress", "message.change.offering.confirm": "Please confirm that you wish to change the service offering of this virtual instance.", "message.change.offering.for.volume": "Successfully changed offering for the volume", @@ -2400,6 +2405,7 @@ "message.confirm.scale.up.system.vm": "Do you really want to scale up the system VM?", "message.confirm.start.lb.vm": "Please confirm you want to start LB VM.", "message.confirm.sync.storage": "Please confirm you want to sync the storage pool", +"message.confirm.type": "To confirm, please type", "message.confirm.upgrade.router.newer.template": "Please confirm that you want to upgrade router to use newer template.", "message.cpu.usage.info": "The CPU usage percentage can exceed 100% if the VM has more than 1 vCPU or when CPU Cap is not enabled. This behavior happens according to the hypervisor being used (e.g: in KVM), due to how they account the stats", "message.create.compute.offering": "Compute offering created", @@ -2534,6 +2540,7 @@ "message.error.cluster.description": "Please enter Kubernetes cluster description.", "message.error.cluster.name": "Please enter cluster name.", "message.error.confirm.password": "Please confirm new password.", +"message.error.confirm.text": "Please enter the confirmation text", "message.error.current.password": "Please enter current password.", "message.error.custom.disk.size": "Please enter custom disk size.", "message.error.date": "Please select a date.", @@ -2776,6 +2783,7 @@ "message.please.wait.while.zone.is.being.created": "Please wait while your zone is being created; this may take a while...", "message.pod.dedicated": "Pod dedicated.", "message.pod.dedication.released": "Pod dedication released.", +"message.prepare.for.shutdown": "Please confirm that you would like to prep this Management server for shutdown. It will not accept any new Async Jobs but will NOT terminate after there are no pending jobs.", "message.primary.storage.invalid.state": "Primary storage is not in Up state", "message.processing.complete": "Processing complete!", "message.protocol.description": "For XenServer, choose NFS, iSCSI, or PreSetup. For KVM, choose NFS, SharedMountPoint, RDB, CLVM or Gluster. For vSphere, choose NFS, PreSetup (VMFS or iSCSI or FiberChannel or vSAN or vVols) or DatastoreCluster. For Hyper-V, choose SMB/CIFS. For LXC, choose NFS or SharedMountPoint. For OVM, choose NFS or OCFS2.", @@ -2851,6 +2859,7 @@ "message.setup.physical.network.during.zone.creation": "When adding a zone, you need to set up one or more physical networks. Each network corresponds to a NIC on the hypervisor. Each physical network can carry one or more types of traffic, with certain restrictions on how they may be combined. Add or remove one or more traffic types onto each physical network.", "message.setup.physical.network.during.zone.creation.basic": "When adding a basic zone, you can set up one physical network, which corresponds to a NIC on the hypervisor. The network carries several types of traffic.

You may also add other traffic types onto the physical network.", "message.shared.network.offering.warning": "Domain admins and regular users can only create shared networks from network offering with the setting specifyvlan=false. Please contact an administrator to create a network offering if this list is empty.", +"message.shutdown.triggered": "A shutdown has been triggered. CloudStack will not accept new jobs", "message.specify.tag.key": "Please specify a tag key.", "message.specify.tag.value": "Please specify a tag value.", "message.step.2.continue": "Please select a service offering to continue.", @@ -2976,6 +2985,7 @@ "message.template.type.change.warning": "WARNING: Changing the template type to SYSTEM will disable further changes to the template.", "message.tooltip.reserved.system.netmask": "The network prefix that defines the pod subnet. Uses CIDR notation.", "message.traffic.type.to.basic.zone": "traffic type to basic zone", +"message.trigger.shutdown": "Please confirm that you would like to trigger a shutdown on this Management server. It will not accept any new Async Jobs and will terminate after there are no pending jobs.", "message.type.values.to.add": "Please add additonal values by typing them in", "message.update.autoscale.policy.failed": "Failed to update autoscale policy", "message.update.autoscale.vmgroup.failed": "Failed to update autoscale vm group", diff --git a/ui/src/components/page/GlobalLayout.vue b/ui/src/components/page/GlobalLayout.vue index 3f8779d60625..97ebcfaeeb9a 100644 --- a/ui/src/components/page/GlobalLayout.vue +++ b/ui/src/components/page/GlobalLayout.vue @@ -16,98 +16,105 @@ // under the License.