diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 7348547cee05..16d61bf89a13 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; @@ -39,6 +40,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; import org.apache.cloudstack.api.response.GetUploadParamsResponse; public interface TemplateApiService { @@ -47,6 +49,10 @@ public interface TemplateApiService { public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplateCmd cmd) throws ResourceAllocationException, MalformedURLException; + GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd); + + VirtualMachineTemplate activateSystemVMTemplate(long templateId); + VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index b7779cb2a8d9..3715e97999e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -299,6 +299,7 @@ public class ApiConstants { public static final String TARGET_IQN = "targetiqn"; public static final String TEMPLATE_FILTER = "templatefilter"; public static final String TEMPLATE_ID = "templateid"; + public static final String SYSTEM = "system"; public static final String ISO_ID = "isoid"; public static final String TIMEOUT = "timeout"; public static final String TIMEZONE = "timezone"; @@ -602,6 +603,7 @@ public class ApiConstants { public static final String INTERVAL = "interval"; public static final String QUIETTIME = "quiettime"; public static final String ACTION = "action"; + public static final String ACTIVATE = "activate"; public static final String CONDITION_ID = "conditionid"; public static final String CONDITION_IDS = "conditionids"; public static final String COUNTERPARAM_LIST = "counterparam"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java new file mode 100644 index 000000000000..36097d67b9f2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.template; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.log4j.Logger; + +import java.util.List; + +@APICommand(name = "activateSystemVMTemplate", description = "Activates an existing system virtual machine template to be used by CloudStack to create system virtual machines.", + responseObject = TemplateResponse.class, authorized = {RoleType.Admin}) +public class ActivateSystemVMTemplateCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(ActivateSystemVMTemplateCmd.class.getName()); + private static final String NAME = "activatesystemvmtemplateresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = TemplateResponse.class, required = true, description = "The template ID of the System VM Template to activate.") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(id); + if (template != null) { + ListResponse response = new ListResponse(); + List templateResponses = _responseGenerator.createTemplateResponses(ResponseObject.ResponseView.Restricted, + template, 1L , false); + response.setResponses(templateResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to activate template."); + } + } + + @Override + public String getCommandName() { + return NAME; + } + + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Template; + } + + @Override + public long getEntityOwnerId() { + final VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, id); + if (template != null) { + return template.getAccountId(); + } + + // bad id given, parent this command to SYSTEM so ERROR events are tracked + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java new file mode 100644 index 000000000000..2057f5278332 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.template; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "getSystemVMTemplateDefaultURL", description = "Gets the system virtual machine template's default download URL.", + responseObject = GetSystemVMTemplateDefaultURLResponse.class, authorized = {RoleType.Admin}) +public class GetSystemVMTemplateDefaultURLCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(GetSystemVMTemplateDefaultURLCmd.class.getName()); + private static final String NAME = "getsystemvmtemplatedefaulturlresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, + entityType = TemplateResponse.class, required = false, description = "The CloudStack version for which to get System VM Template URL.") + private String version; + + @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, entityType = TemplateResponse.class, required = true, description = "The hypervisor for which to get System VM Template URL.") + private String hypervisor; + + public String getVersion() { + return version; + } + + public String getHypervisor() { + return hypervisor; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + GetSystemVMTemplateDefaultURLResponse response = _templateService.getSystemVMTemplateDefaultURL(this); + if (response != null) { + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to find URL for version '%s' and hypervisor '%s'", version, hypervisor)); + } + } + + @Override + public String getCommandName() { + return NAME; + } + + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Template; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 333b363d16a9..e7b51d590713 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; @@ -161,6 +162,13 @@ public class RegisterTemplateCmd extends BaseCmd { description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment") private Boolean directDownload; + @Parameter(name = ApiConstants.SYSTEM, type = CommandType.BOOLEAN, description = "true if it is a system vm template.") + private Boolean system; + + @Parameter(name=ApiConstants.ACTIVATE, type = CommandType.BOOLEAN, + description = "true if this template should be used by CloudStack to create System VMs. Must be used with template type of 'system'.") + private Boolean activate; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -273,6 +281,14 @@ public boolean isDirectDownload() { return directDownload == null ? false : directDownload; } + public Boolean isSystem() { + return Optional.ofNullable(system).orElse(false); + } + + public Boolean isActivate() { + return Optional.ofNullable(activate).orElse(false); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -323,7 +339,7 @@ protected void validateParameters() { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Both zoneid and zoneids cannot be specified at the same time"); - if (zoneId == null && (zoneIds == null || zoneIds.isEmpty())) + if ((zoneId == null && (zoneIds == null || zoneIds.isEmpty())) && !isSystem()) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Either zoneid or zoneids is required. Both cannot be null."); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java new file mode 100644 index 000000000000..272bfdfb8605 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +public class GetSystemVMTemplateDefaultURLResponse extends BaseResponse { + + @SerializedName(ApiConstants.URL) + @Param(description = "The default URL of the System VM template.") + private String url; + + public GetSystemVMTemplateDefaultURLResponse() { + } + + public GetSystemVMTemplateDefaultURLResponse(String responseName, String url) { + setResponseName(responseName); + setObjectName("url"); + this.url = url; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java index 25c2d5b3c074..55be0b2c409d 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java @@ -26,7 +26,7 @@ public final class TemplateConstants { public static final String DEFAULT_TMPLT_FIRST_LEVEL_DIR = "tmpl/"; public static final String DEFAULT_SYSTEM_VM_TEMPLATE_PATH = "template/tmpl/1/"; - + public static final String DEFAULT_BASE_SYSTEMVM_URL = "https://download.cloudstack.org/systemvm"; public static final String DEFAULT_SYSTEM_VM_TMPLT_NAME = "routing"; public static final int DEFAULT_TMPLT_COPY_PORT = 80; diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index fc8a769e81f9..abad08aeb795 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -51,6 +51,8 @@ public TemplateInfo getTemplate() { AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, DataStore destStore); + AsyncCallFuture copySystemVMTemplate(TemplateInfo srcTemplate, DataStore destStore); + AsyncCallFuture prepareTemplateOnPrimary(TemplateInfo srcTemplate, StoragePool pool); AsyncCallFuture deleteTemplateOnPrimary(TemplateInfo template, StoragePool pool); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index c43a2ea4ee9e..a0296ba23478 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -40,6 +40,8 @@ public interface VMTemplateDao extends GenericDao, StateDao< public List listAllSystemVMTemplates(); + List listSystemVMTemplatesByUrlLike(String partialUrl, String hType); + public List listDefaultBuiltinTemplates(); public String getRoutingTemplateUniqueName(); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index dd1f2fcf1641..8a6d6398b8d6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -92,6 +92,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem protected SearchBuilder tmpltTypeHyperSearch; protected SearchBuilder readySystemTemplateSearch; protected SearchBuilder tmpltTypeHyperSearch2; + protected SearchBuilder urlLikeSearch; protected SearchBuilder AccountIdSearch; protected SearchBuilder NameSearch; @@ -221,6 +222,15 @@ public List listAllSystemVMTemplates() { return listBy(sc, filter); } + @Override + public List listSystemVMTemplatesByUrlLike(String partialUrl, String hypervisorType) { + SearchCriteria sc = urlLikeSearch.create(); + sc.setParameters("templateType", TemplateType.SYSTEM); + sc.setParameters("hypervisorType", hypervisorType); + sc.setParameters("url", "%" + partialUrl + "%"); + return listBy(sc); + } + @Override public List listPrivateTemplatesByHost(Long hostId) { @@ -417,6 +427,12 @@ public boolean configure(String name, Map params) throws Configu ParentTemplateIdSearch.and("state", ParentTemplateIdSearch.entity().getState(), SearchCriteria.Op.EQ); ParentTemplateIdSearch.done(); + urlLikeSearch = createSearchBuilder(); + urlLikeSearch.and("templateType", urlLikeSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); + urlLikeSearch.and("hypervisorType", urlLikeSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + urlLikeSearch.and("url", urlLikeSearch.entity().getUrl(), SearchCriteria.Op.LIKE); + urlLikeSearch.done(); + return result; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 6fda4a15c324..47c666b0b8ff 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; @@ -82,6 +83,8 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listByTypes(VirtualMachine.Type... types); + List listByHypervisorTypeAndNonUserTypes(HypervisorType hypervisorType); + VMInstanceVO findByIdTypes(long id, VirtualMachine.Type... types); VMInstanceVO findVMByInstanceName(String name); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 1565f53233bc..1d5ae7880071 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -28,6 +28,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -77,6 +78,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder StateChangeSearch; protected SearchBuilder TransitionSearch; protected SearchBuilder TypesSearch; + protected SearchBuilder hypervisorTypeAndNonUserTypeSearch; protected SearchBuilder IdTypesSearch; protected SearchBuilder HostIdTypesSearch; protected SearchBuilder HostIdStatesSearch; @@ -193,6 +195,12 @@ protected void init() { TypesSearch.and("types", TypesSearch.entity().getType(), Op.IN); TypesSearch.done(); + hypervisorTypeAndNonUserTypeSearch = createSearchBuilder(); + hypervisorTypeAndNonUserTypeSearch.and("hypervisorType", + hypervisorTypeAndNonUserTypeSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + hypervisorTypeAndNonUserTypeSearch.and("types", hypervisorTypeAndNonUserTypeSearch.entity().getType(), Op.NIN); + hypervisorTypeAndNonUserTypeSearch.done(); + IdTypesSearch = createSearchBuilder(); IdTypesSearch.and("id", IdTypesSearch.entity().getId(), Op.EQ); IdTypesSearch.and("types", IdTypesSearch.entity().getType(), Op.IN); @@ -427,6 +435,14 @@ public List listByTypes(Type... types) { return listBy(sc); } + @Override + public List listByHypervisorTypeAndNonUserTypes(HypervisorType hypervisorType) { + SearchCriteria sc = hypervisorTypeAndNonUserTypeSearch.create(); + sc.setParameters("hypervisorType", hypervisorType); + sc.setParameters("types", Type.User); + return listBy(sc); + } + @Override public List listByTypeAndState(VirtualMachine.Type type, State state) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 84c27583925b..6238484deef6 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -356,4 +356,5 @@ + diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 45e3941a5ec3..37ab7accbdd1 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; +import com.cloud.storage.copy.ManagementServerCopier; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; @@ -156,6 +157,8 @@ public class TemplateServiceImpl implements TemplateService { ImageStoreDetailsUtil imageStoreDetailsUtil; @Inject TemplateDataFactory imageFactory; + @Inject + private ManagementServerCopier managementServerCopier; class TemplateOpContext extends AsyncRpcContext { final TemplateObject template; @@ -1017,6 +1020,9 @@ public void syncTemplateToRegionStore(long templateId, DataStore store) { @Override public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, DataStore destStore) { + if (srcTemplate.getTemplateType() == TemplateType.SYSTEM) { + return copySystemVMTemplate(srcTemplate, destStore); + } // for vmware template, we need to check if ova packing is needed, since template created from snapshot does not have .ova file // we invoke createEntityExtractURL to trigger ova packing. Ideally, we can directly use extractURL to pass to following createTemplate. // Need to understand what is the background to use two different urls for copy and extract. @@ -1055,11 +1061,7 @@ public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, destStore.getDriver().createAsync(destStore, templateOnStore, caller); } catch (CloudRuntimeException ex) { // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called - TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), srcTemplate.getId()); - if (templateStoreVO != null) { - TemplateInfo tmplObj = _templateFactory.getTemplate(srcTemplate, destStore); - tmplObj.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); - } + cleanupTemplateStoreRefEntry(srcTemplate, destStore); TemplateApiResult res = new TemplateApiResult((TemplateObject)templateOnStore); res.setResult(ex.getMessage()); future.complete(res); @@ -1067,6 +1069,32 @@ public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, return future; } + @Override + public AsyncCallFuture copySystemVMTemplate(TemplateInfo srcTemplate, DataStore destStore) { + AsyncCallFuture future = new AsyncCallFuture<>(); + TemplateObject tmplForCopy = (TemplateObject)_templateFactory.getTemplate(srcTemplate, destStore); + DataObject templateOnStore = destStore.create(tmplForCopy); + templateOnStore.processEvent(Event.CreateOnlyRequested); + TemplateApiResult res = new TemplateApiResult(new TemplateObject()); + try { + res.setSuccess(managementServerCopier.copy(srcTemplate, destStore)); + } catch (CloudRuntimeException e) { + cleanupTemplateStoreRefEntry(srcTemplate, destStore); + res.setSuccess(false); + res.setResult(e.getMessage()); + } + future.complete(res); + return future; + } + + private void cleanupTemplateStoreRefEntry(TemplateInfo srcTemplate, DataStore destStore) { + TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), srcTemplate.getId()); + if (templateStoreVO != null) { + TemplateInfo info = _templateFactory.getTemplate(srcTemplate, destStore); + info.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + } + } + private String generateCopyUrl(String ipAddress, String dir, String path) { String hostname = ipAddress; String scheme = "http"; diff --git a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml index 33385b5ae001..f512272f862d 100644 --- a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml +++ b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml @@ -50,6 +50,9 @@ + + diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 4cd2811cc204..a432744a8332 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -206,10 +206,12 @@ import org.apache.cloudstack.api.command.admin.systemvm.UpgradeSystemVMCmd; import org.apache.cloudstack.api.command.admin.template.CopyTemplateCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.CreateTemplateCmdByAdmin; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.admin.template.ListTemplatePermissionsCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.ListTemplatesCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.PrepareTemplateCmd; import org.apache.cloudstack.api.command.admin.template.RegisterTemplateCmdByAdmin; +import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; import org.apache.cloudstack.api.command.admin.usage.AddTrafficMonitorCmd; import org.apache.cloudstack.api.command.admin.usage.AddTrafficTypeCmd; import org.apache.cloudstack.api.command.admin.usage.DeleteTrafficMonitorCmd; @@ -2842,6 +2844,8 @@ public List> getCommands() { cmdList.add(ListTemplatePermissionsCmd.class); cmdList.add(ListTemplatesCmd.class); cmdList.add(RegisterTemplateCmd.class); + cmdList.add(ActivateSystemVMTemplateCmd.class); + cmdList.add(GetSystemVMTemplateDefaultURLCmd.class); cmdList.add(UpdateTemplateCmd.class); cmdList.add(UpdateTemplatePermissionsCmd.class); cmdList.add(AddNicToVMCmd.class); diff --git a/server/src/main/java/com/cloud/storage/copy/ManagementServerCopier.java b/server/src/main/java/com/cloud/storage/copy/ManagementServerCopier.java new file mode 100644 index 000000000000..bc9b75662a53 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/copy/ManagementServerCopier.java @@ -0,0 +1,10 @@ +package com.cloud.storage.copy; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; + + +public interface ManagementServerCopier { + + boolean copy(TemplateInfo srcTemplate, DataStore destStore); +} diff --git a/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java b/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java new file mode 100644 index 000000000000..5f26b3ce5fb8 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java @@ -0,0 +1,99 @@ +package com.cloud.storage.copy; + +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.mount.MountManager; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Date; +import java.util.Optional; + +@Component +public class NfsManagementServerCopier implements ManagementServerCopier { + private static final Logger s_logger = Logger.getLogger(NfsManagementServerCopier.class); + + @Inject + private MountManager mountManager; + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; + @Inject + private TemplateDataStoreDao vmTemplateStoreDao; + + + public boolean copy(TemplateInfo srcTemplate, DataStore destStore) { + Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(srcTemplate.getDataStore().getId()); + String sourceMountPoint = mountManager.getMountPoint(srcTemplate.getDataStore().getUri(), nfsVersion); + String destMountPoint = mountManager.getMountPoint(destStore.getUri(), nfsVersion); + String installPath = getTemplateDataStore(srcTemplate).getInstallPath(); + String sourcePath = Paths.get(sourceMountPoint, installPath).toString(); + String destinationPath = Paths.get(destMountPoint, installPath).toString(); + createFolder(Paths.get(destMountPoint, installPath)); + if (copyFolderContent(sourcePath, destinationPath)) { + persistTemplateStoreRef(srcTemplate, destStore, destinationPath); + return true; + } + return false; + } + + private boolean copyFolderContent(String sourcePath, String destinationPath) { + String command = String.format("cp %s%s* %s", sourcePath, File.separator, destinationPath); + String result = Script.runSimpleBashScript(command); + if (result != null) { + s_logger.warn(String.format("Unable to copy from %s to %s due to %s.%n", sourcePath, destinationPath, result)); + return false; + } + return true; + } + + /** + * Create folder on path if it does not exist + */ + public static void createFolder(Path path) { + try { + Files.createDirectories(path); + } catch (IOException e) { + s_logger.error(String.format("Unable to create directory from path %s: %s.%n", path, e.toString())); + throw new CloudRuntimeException(e); + } + } + + private TemplateDataStoreVO getTemplateDataStore(TemplateInfo template) { + return Optional.ofNullable(vmTemplateStoreDao.findByStoreTemplate(template.getDataStore().getId(), template.getId())) + .orElseThrow(() -> new CloudRuntimeException(String.format("Unable to find template store ref by template id %d.%n", template.getId()))); + } + + private void persistTemplateStoreRef(TemplateInfo template, DataStore destStore, String destinationPath) { + TemplateDataStoreVO vmTemplateStore = vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), template.getId()); + if (vmTemplateStore == null) { + vmTemplateStore = new TemplateDataStoreVO(destStore.getId(), template.getId(), new Date(), 100, + VMTemplateStorageResourceAssoc.Status.DOWNLOADED, template.getInstallPath(), null, null, template.getInstallPath(), template.getUri()); + vmTemplateStore.setDataStoreRole(destStore.getRole()); + vmTemplateStoreDao.persist(vmTemplateStore); + } else { + vmTemplateStore.setDownloadPercent(100); + vmTemplateStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + vmTemplateStore.setSize(Optional.ofNullable(template.getSize()).orElse(0L)); + vmTemplateStore.setPhysicalSize(Optional.ofNullable(template.getSize()).orElse(0L)); + vmTemplateStore.setLastUpdated(new Date()); + vmTemplateStore.setInstallPath(template.getInstallPath()); + vmTemplateStore.setLocalDownloadPath(destinationPath); + vmTemplateStore.setDownloadUrl(template.getUri()); + vmTemplateStore.setState(ObjectInDataStoreStateMachine.State.Ready); + vmTemplateStoreDao.update(vmTemplateStore.getId(), vmTemplateStore); + } + } +} diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 68be52d842fb..7f0780d9a54f 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -16,18 +16,25 @@ // under the License. package com.cloud.storage.download; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Timer; - -import javax.inject.Inject; - -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.configuration.Config; +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.RegisterVolumePayload; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.Volume; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.download.managementserver.ManagementServerDownloader; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.storage.upload.UploadListener; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -45,23 +52,17 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; -import com.cloud.agent.AgentManager; -import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; -import com.cloud.configuration.Config; -import com.cloud.storage.RegisterVolumePayload; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import com.cloud.storage.Volume; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.storage.template.TemplateConstants; -import com.cloud.storage.upload.UploadListener; -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.utils.component.ComponentContext; -import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Timer; @Component public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor { @@ -81,11 +82,16 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor private ConfigurationDao _configDao; @Inject private EndPointSelector _epSelector; + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; + @Inject + private ManagementServerDownloader managementServerDownloader; private String _copyAuthPasswd; private String _proxy = null; private Timer _timer; + private final Map _storageMounts = new HashMap<>(); @Override public boolean configure(String name, Map params) { @@ -152,28 +158,34 @@ private void initiateTemplateDownload(DataObject template, AsyncCompletionCallba } EndPoint ep = _epSelector.select(template); if (ep == null) { - String errMsg = "There is no secondary storage VM for downloading template to image store " + store.getName(); - s_logger.warn(errMsg); - throw new CloudRuntimeException(errMsg); - } - DownloadListener dl = new DownloadListener(ep, store, template, _timer, this, dcmd, callback); - ComponentContext.inject(dl); // initialize those auto-wired field in download listener. - if (downloadJobExists) { - // due to handling existing download job issues, we still keep - // downloadState in template_store_ref to avoid big change in - // DownloadListener to use - // new ObjectInDataStore.State transition. TODO: fix this later - // to be able to remove downloadState from template_store_ref. - s_logger.info("found existing download job"); - dl.setCurrState(vmTemplateStore.getDownloadState()); - } + if (tmpl.getTemplateType() == Storage.TemplateType.SYSTEM) { + managementServerDownloader.download(tmpl, template); + //set to ready state + } else { + String errMsg = "There is no secondary storage VM for downloading template to image store " + store.getName(); + s_logger.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } + } else { + DownloadListener dl = new DownloadListener(ep, store, template, _timer, this, dcmd, callback); + ComponentContext.inject(dl); // initialize those auto-wired field in download listener. + if (downloadJobExists) { + // due to handling existing download job issues, we still keep + // downloadState in template_store_ref to avoid big change in + // DownloadListener to use + // new ObjectInDataStore.State transition. TODO: fix this later + // to be able to remove downloadState from template_store_ref. + s_logger.info("found existing download job"); + dl.setCurrState(vmTemplateStore.getDownloadState()); + } - try { - ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); - } catch (Exception e) { - s_logger.warn("Unable to start /resume download of template " + template.getId() + " to " + store.getName(), e); - dl.setDisconnected(); - dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + try { + ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); + } catch (Exception e) { + s_logger.warn("Unable to start /resume download of template " + template.getId() + " to " + store.getName(), e); + dl.setDisconnected(); + dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + } } } } diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java new file mode 100644 index 000000000000..0bbd8a09a947 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java @@ -0,0 +1,129 @@ +package com.cloud.storage.download.managementserver; + +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.download.managementserver.system.HttpSystemTemplateDownloader; +import com.cloud.storage.mount.MountManager; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class ManagementServerDownloader { + private static final Logger s_logger = Logger.getLogger(ManagementServerDownloader.class); + + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; + + @Inject + VMTemplatePoolDao vmTemplatePoolDao; + + @Inject + private VMTemplateDao templateDao; + + @Inject + private TemplateDataStoreDao vmTemplateStoreDao; + + @Inject + private MountManager mountManager; + + private static final ExecutorService THREAD_SERVICE = Executors.newCachedThreadPool(new NamedThreadFactory("ManagementServerDownloader")); + + public void download(VirtualMachineTemplate tmpl, DataObject template) { + downloadFromUrlToNfs(tmpl, template, template.getUuid()); + } + + + public void downloadFromUrlToNfs(VirtualMachineTemplate tmpl, DataObject template, String fileName) { + Runnable downloadFile = () -> { + Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(template.getDataStore().getId()); + String mountPoint = mountManager.getMountPoint(template.getDataStore().getUri(), nfsVersion); + VMTemplateVO templateVO = Optional.ofNullable(templateDao.findById(template.getId())) + .orElseThrow(() -> new CloudRuntimeException(String.format("Unable to find template by id %d.%n", template.getId()))); + HttpSystemTemplateDownloader downloader = new HttpSystemTemplateDownloader(tmpl, templateVO, mountPoint); + if (downloader.downloadTemplate()) { + if (downloader.extractAndInstallDownloadedTemplate()) { + TemplateDownloader.TemplateInformation info = downloader.getTemplateInformation(); + try(TransactionLegacy txn = TransactionLegacy.open(ManagementServerDownloader.class.getName())) { + persistTemplate(template, templateVO, info); + persistTemplateStorePoolRef(tmpl, template.getDataStore().getId(), info); + persistTemplateStoreRef(template, info); + } + } else { + throw new CloudRuntimeException(String.format("Failed to extract template %s from url %s", fileName, tmpl.getUrl())); + } + } else { + throw new CloudRuntimeException(String.format("Failed to download template %s from url %s", fileName, tmpl.getUrl())); + } + }; + THREAD_SERVICE.submit(downloadFile); + } + + private void persistTemplate(DataObject template, VMTemplateVO templateVO, TemplateDownloader.TemplateInformation info) { + templateVO.setSize(info.getSize()); + templateDao.update(template.getId(), templateVO); + } + + private void persistTemplateStoreRef(DataObject template, TemplateDownloader.TemplateInformation info) { + DataStore store = template.getDataStore(); + TemplateDataStoreVO vmTemplateStore = vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); + if (vmTemplateStore == null) { + vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 100, + VMTemplateStorageResourceAssoc.Status.DOWNLOADED, info.getInstallPath(), null, null, info.getInstallPath(), template.getUri()); + vmTemplateStore.setDataStoreRole(store.getRole()); + vmTemplateStoreDao.persist(vmTemplateStore); + } else { + vmTemplateStore.setDownloadPercent(100); + vmTemplateStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + vmTemplateStore.setSize(info.getSize()); + vmTemplateStore.setPhysicalSize(info.getSize()); + vmTemplateStore.setLastUpdated(new Date()); + vmTemplateStore.setInstallPath(info.getInstallPath()); + vmTemplateStore.setLocalDownloadPath(info.getLocalPath()); + vmTemplateStore.setDownloadUrl(template.getUri()); + vmTemplateStore.setState(ObjectInDataStoreStateMachine.State.Ready); + vmTemplateStoreDao.update(vmTemplateStore.getId(), vmTemplateStore); + } + } + + private void persistTemplateStorePoolRef(VirtualMachineTemplate tmpl, long poolId, TemplateDownloader.TemplateInformation info) { + VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, tmpl.getId()); + if (sPoolRef == null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Not found (templateId:" + tmpl.getId() + " poolId: " + poolId + ") in template_spool_ref, persisting it"); + } + sPoolRef = new VMTemplateStoragePoolVO(poolId, tmpl.getId()); + sPoolRef.setDownloadPercent(100); + sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); + sPoolRef.setTemplateSize(info.getSize()); + sPoolRef.setLocalDownloadPath(info.getLocalPath()); + sPoolRef.setInstallPath(info.getInstallPath()); + vmTemplatePoolDao.persist(sPoolRef); + } else { + sPoolRef.setDownloadPercent(100); + sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); + sPoolRef.setTemplateSize(info.getSize()); + sPoolRef.setLocalDownloadPath(info.getLocalPath()); + sPoolRef.setInstallPath(info.getInstallPath()); + vmTemplatePoolDao.update(sPoolRef.getId(), sPoolRef); + } + } +} diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloader.java new file mode 100644 index 000000000000..11e890b9584d --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloader.java @@ -0,0 +1,77 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.storage.download.managementserver; + +public interface TemplateDownloader { + + class TemplateInformation { + private String installPath; + private String localPath; + private Long size; + private String checksum; + + public TemplateInformation(String installPath, String localPath, Long size, String checksum) { + this.installPath = installPath; + this.localPath = localPath; + this.size = size; + this.checksum = checksum; + } + + public String getInstallPath() { + return installPath; + } + + public String getLocalPath() { + return localPath; + } + + public Long getSize() { + return size; + } + + public String getChecksum() { + return checksum; + } + } + + /** + * Perform template download to pool specified on downloader creation + * @return true if successful, false if not + */ + boolean downloadTemplate(); + + /** + * Perform extraction (if necessary) and installation of previously downloaded template + * @return true if successful, false if not + */ + boolean extractAndInstallDownloadedTemplate(); + + /** + * Get template information after it is properly installed on pool + * @return template information + */ + TemplateInformation getTemplateInformation(); + + /** + * Perform checksum validation of previously downloadeed template + * @return true if successful, false if not + */ + boolean validateChecksum(); +} diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/system/HttpSystemTemplateDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/system/HttpSystemTemplateDownloader.java new file mode 100644 index 000000000000..3a10c9af2e86 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/managementserver/system/HttpSystemTemplateDownloader.java @@ -0,0 +1,81 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.storage.download.managementserver.system; + +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +public class HttpSystemTemplateDownloader extends SystemTemplateDownloader { + + @Inject + private VMTemplateDao _templateDao; + + protected HttpClient client; + public static final Logger logger = Logger.getLogger(HttpSystemTemplateDownloader.class.getName()); + + public HttpSystemTemplateDownloader(VirtualMachineTemplate template,VMTemplateVO templateVO, String destPoolPath) { + super(template, templateVO, destPoolPath); + String downloadDir = getDownloadPath(template.getId()); + createDownloadDirectory(downloadDir); + setDownloadedFilePath(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); + } + + protected void createDownloadDirectory(String downloadDir) { + createFolder(getDestPoolPath() + File.separator + downloadDir); + } + + public boolean downloadTemplate() { + HttpClient client = new DefaultHttpClient(); + HttpGet get = new HttpGet(getTemplate().getUrl()); + try { + HttpResponse response = client.execute(get); + org.apache.http.HttpEntity entity = response.getEntity(); + if (entity == null) { + s_logger.debug("Failed to get entity"); + throw new CloudRuntimeException("Failed to get url: " + getTemplate().getUrl()); + } + File destFile = new File(getDownloadedFilePath()); + if (!destFile.createNewFile()) { + s_logger.warn("Reusing existing file " + destFile.getPath()); + } + try (FileOutputStream outputStream = new FileOutputStream(destFile)) { + entity.writeTo(outputStream); + } catch (IOException e) { + s_logger.debug("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); + } + } catch (IOException e) { + s_logger.debug("Failed to get url:" + getTemplate().getUrl() + ", due to " + e.toString()); + throw new CloudRuntimeException(e); + } + return true; + } +} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/system/SystemTemplateDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/system/SystemTemplateDownloader.java new file mode 100644 index 000000000000..b68237924ec3 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/managementserver/system/SystemTemplateDownloader.java @@ -0,0 +1,267 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.storage.download.managementserver.system; + +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.download.managementserver.TemplateDownloader; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.util.Optional; +import java.util.UUID; + +public abstract class SystemTemplateDownloader implements TemplateDownloader { + + private String destPoolPath; + private String downloadedFilePath; + private String installPath; + private VirtualMachineTemplate template; + private VMTemplateVO templateVO; + private boolean redownload = false; + private String srcHost; + private String srcPath; + + public static final Logger s_logger = Logger.getLogger(SystemTemplateDownloader.class.getName()); + private static String systemInstallDir = "template" + File.separator + "tmpl" + File.separator + "1"; + private static final String systemPropertiesFileName = "template.properties"; + + protected SystemTemplateDownloader(VirtualMachineTemplate template, VMTemplateVO templateVO, String destPoolPath) { + this.template = template; + this.templateVO = templateVO; + this.destPoolPath = destPoolPath; + parseUrl(); + } + + /** + * Return download path to download template + */ + protected static String getDownloadPath(Long templateId) { + return systemInstallDir + File.separator + templateId; + } + + /** + * Create folder on path if it does not exist + */ + protected void createFolder(String path) { + try { + Files.createDirectories(Paths.get(path)); + } catch (IOException e) { + s_logger.error(String.format("Unable to create directory %s for template %s: %s%n", path, getTemplate().getUrl(), e.toString())); + throw new CloudRuntimeException(e); + } + } + + public String getDestPoolPath() { + return destPoolPath; + } + + + public String getDownloadedFilePath() { + return downloadedFilePath; + } + + public void setDownloadedFilePath(String filePath) { + this.downloadedFilePath = filePath; + } + + public VirtualMachineTemplate getTemplate() { + return template; + } + + public boolean isRedownload() { + return redownload; + } + + /** + * Return filename from url + */ + public String getFileNameFromUrl() { + String[] urlParts = template.getUrl().split("/"); + return urlParts[urlParts.length - 1]; + } + + public String getInstallFileName() { + return template.getUuid() + "." + template.getFormat().toString().toLowerCase(); + } + + /** + * Checks if downloaded template is extractable + * @return true if it should be extracted, false if not + */ + private boolean isTemplateExtractable() { + String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'"); + return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); + } + + @Override + public boolean extractAndInstallDownloadedTemplate() { + installPath = UUID.randomUUID().toString(); + createFolder(getInstallFullPath()); + if (isTemplateExtractable()) { + extractDownloadedTemplate(); + } else { + s_logger.info(String.format("The downloaded template from %s is not extractable.%n", template.getUrl())); + } + createPropertiesFile(); + return true; + } + + private void createPropertiesFile() { + try { + Path propertiesFile = Files.createFile(Paths.get(getInstallFullPath() + File.separator + systemPropertiesFileName)); + Files.write(propertiesFile, getTemplatePropertiesFileContent().getBytes()); + } catch (IOException e) { + String errorMessage = String.format("Unable to create %s file for downloaded template %s: %s.%n", systemPropertiesFileName, template.getUrl(), e.getMessage()); + s_logger.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + } + + private String getTemplatePropertiesFileContent() { + TemplateInformation info = getTemplateInformation(); + StringBuffer content = new StringBuffer(500); + content.append(String.format("filename=%s%n", getInstallFileName())); + content.append(String.format("description=%s%n", template.getDisplayText())); + content.append(String.format("checksum=%s%n", Optional.ofNullable(template.getChecksum()).orElse(""))); + content.append(String.format("hvm=%s%n", template.isRequiresHvm())); + content.append(String.format("size=%s%n", info.getSize())); + content.append(String.format("%s=true%n", template.getFormat().getFileExtension())); + content.append(String.format("id=%s%n", template.getId())); + content.append(String.format("public=%s%n", template.isPublicTemplate())); + content.append(String.format("%s.filename=%s%n", template.getFormat().getFileExtension(), getInstallFileName())); + content.append(String.format("uniquename=%s%n", templateVO.getUniqueName())); + content.append(String.format("%s.virtualsize=%s%n", template.getFormat().getFileExtension(), info.getSize())); + content.append(String.format("%s.size=%s%n", template.getFormat().getFileExtension(), info.getSize())); + return content.toString(); + } + + /** + * Return install full path + */ + private String getInstallFullPath() { + return destPoolPath + File.separator + systemInstallDir + File.separator + String.valueOf(template.getId()); + } + + /** + * Return extract command to execute given downloaded file + */ + private String getExtractCommandForDownloadedFile() { + if (downloadedFilePath.endsWith(".zip")) { + return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath() + File.separator + getInstallFileName(); + } else if (downloadedFilePath.endsWith(".bz2")) { + return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath() + File.separator + getInstallFileName(); + } else if (downloadedFilePath.endsWith(".gz")) { + return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath() + File.separator + getInstallFileName(); + } else { + throw new CloudRuntimeException("Unable to extract template " + template.getId() + " on " + downloadedFilePath); + } + } + + /** + * Extract downloaded template into installPath, remove compressed file + */ + private void extractDownloadedTemplate() { + String extractCommand = getExtractCommandForDownloadedFile(); + Script.runSimpleBashScript(extractCommand); + Script.runSimpleBashScript("rm -f " + downloadedFilePath); + } + + @Override + public TemplateInformation getTemplateInformation() { + String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + File.separator + getInstallFileName() + " | awk '{print $1}'"); + long size = Long.parseLong(sizeResult); + return new TemplateInformation(systemInstallDir + File.separator + String.valueOf(template.getId()), + getTemplate().getUuid(), size, template.getChecksum()); + } + + @Override + public boolean validateChecksum() { + if (StringUtils.isNotBlank(template.getChecksum())) { + int retry = 3; + boolean valid = false; + try { + while (!valid && retry > 0) { + retry--; + s_logger.info("Performing checksum validation for downloaded template " + template.getId() + " using " + template.getChecksum() + ", retries left: " + retry); + valid = DigestHelper.check(template.getChecksum(), new FileInputStream(downloadedFilePath)); + if (!valid && retry > 0) { + s_logger.info("Checksum validation failded, re-downloading template"); + redownload = true; + resetDownloadFile(); + downloadTemplate(); + } + } + s_logger.info("Checksum validation for template " + template.getId() + ": " + (valid ? "succeeded" : "failed")); + return valid; + } catch (IOException e) { + throw new CloudRuntimeException("could not check sum for file: " + downloadedFilePath, e); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException("Unknown checksum algorithm: " + template.getChecksum(), e); + } + } + s_logger.info("No checksum provided, skipping checksum validation"); + return true; + } + + /** + * Delete and create download file + */ + private void resetDownloadFile() { + File f = new File(getDownloadedFilePath()); + s_logger.info("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + template.getId() + " on it"); + try { + if (f.exists()) { + f.delete(); + } + f.createNewFile(); + } catch (IOException e) { + s_logger.error("Error creating file to download on: " + getDownloadedFilePath() + " due to: " + e.getMessage()); + throw new CloudRuntimeException("Failed to create download file for direct download"); + } + } + + /** + * Parse url and set srcHost and srcPath + */ + private void parseUrl() { + try { + URI uri = new URI(UriUtils.encodeURIComponent(template.getUrl())); + if (uri.getScheme() != null && uri.getScheme().equalsIgnoreCase("nfs")) { + srcHost = uri.getHost(); + srcPath = uri.getPath(); + } + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Invalid NFS url " + template.getUrl() + " caused error: " + e.getMessage()); + } + } +} diff --git a/server/src/main/java/com/cloud/storage/mount/MountManager.java b/server/src/main/java/com/cloud/storage/mount/MountManager.java new file mode 100644 index 000000000000..e306cbaf7cd8 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/mount/MountManager.java @@ -0,0 +1,5 @@ +package com.cloud.storage.mount; + +public interface MountManager { + String getMountPoint(String storageUrl, Integer nfsVersion); +} diff --git a/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java b/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java new file mode 100644 index 000000000000..62dd7f545623 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java @@ -0,0 +1,185 @@ +package com.cloud.storage.mount; + +import com.cloud.storage.StorageLayer; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.annotation.PreDestroy; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Component +public class NfsMountManager implements MountManager { + private static final Logger s_logger = Logger.getLogger(NfsMountManager.class); + + private StorageLayer storage; + private int timeout; + private final Random rand = new Random(System.currentTimeMillis()); + private final ConcurrentMap storageMounts = new ConcurrentHashMap<>(); + + public static final ConfigKey MOUNT_PARENT = new ConfigKey<>("Advanced", String.class, + "mount.parent", "/var/cloudstack/mnt", + "The mount point on the Management Server for Secondary Storage.", + true, ConfigKey.Scope.Global); + + public NfsMountManager(StorageLayer storage, int timeout) { + this.storage = storage; + this.timeout = timeout; + } + + public String getMountPoint(String storageUrl, Integer nfsVersion) { + String mountPoint = storageMounts.get(storageUrl); + if (mountPoint != null) { + return mountPoint; + } + + URI uri; + try { + uri = new URI(storageUrl); + } catch (URISyntaxException e) { + s_logger.error("Invalid storage URL format ", e); + throw new CloudRuntimeException("Unable to create mount point due to invalid storage URL format " + storageUrl); + } + + mountPoint = mount(uri.getHost() + ":" + uri.getPath(), MOUNT_PARENT.value(), nfsVersion); + if (mountPoint == null) { + s_logger.error("Unable to create mount point for " + storageUrl); + throw new CloudRuntimeException("Unable to create mount point for " + storageUrl); + } + + storageMounts.putIfAbsent(storageUrl, mountPoint); + return mountPoint; + } + + private String mount(String path, String parent, Integer nfsVersion) { + String mountPoint = setupMountPoint(parent); + if (mountPoint == null) { + s_logger.warn("Unable to create a mount point"); + return null; + } + + Script command = new Script(true, "mount", timeout, s_logger); + command.add("-t", "nfs"); + if (nfsVersion != null){ + command.add("-o", "vers=" + nfsVersion); + } + // command.add("-o", "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0"); + if ("Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) { + command.add("-o", "resvport"); + } + command.add(path); + command.add(mountPoint); + String result = command.execute(); + if (result != null) { + s_logger.warn("Unable to mount " + path + " due to " + result); + deleteMountPath(mountPoint); + return null; + } + + // Change permissions for the mountpoint + Script script = new Script(true, "chmod", timeout, s_logger); + script.add("1777", mountPoint); + result = script.execute(); + if (result != null) { + s_logger.warn("Unable to set permissions for " + mountPoint + " due to " + result); + } + return mountPoint; + } + + private String setupMountPoint(String parent) { + String mountPoint = null; + for (int i = 0; i < 10; i++) { + String mntPt = parent + File.separator + String.valueOf(ManagementServerNode.getManagementServerId()) + "." + Integer.toHexString(rand.nextInt(Integer.MAX_VALUE)); + File file = new File(mntPt); + if (!file.exists()) { + if (storage.mkdir(mntPt)) { + mountPoint = mntPt; + break; + } + } + s_logger.error("Unable to create mount: " + mntPt); + } + + return mountPoint; + } + + private void umount(String localRootPath) { + if (!mountExists(localRootPath)) { + return; + } + Script command = new Script(true, "umount", timeout, s_logger); + command.add(localRootPath); + String result = command.execute(); + if (result != null) { + // Fedora Core 12 errors out with any -o option executed from java + String errMsg = "Unable to umount " + localRootPath + " due to " + result; + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + deleteMountPath(localRootPath); + s_logger.debug("Successfully umounted " + localRootPath); + } + + private void deleteMountPath(String localRootPath) { + try { + Files.deleteIfExists(Paths.get(localRootPath)); + } catch (IOException e) { + s_logger.warn(String.format("unable to delete mount directory %s:%s.%n", localRootPath, e.getMessage())); + } + } + + private boolean mountExists(String localRootPath) { + Script script = new Script(true, "mount", timeout, s_logger); + ZfsPathParser parser = new ZfsPathParser(localRootPath); + script.execute(parser); + return parser.getPaths().stream().filter(s -> s.contains(localRootPath)).findAny().map(s -> true).orElse(false); + } + + public static class ZfsPathParser extends OutputInterpreter { + String _parent; + List paths = new ArrayList<>(); + + public ZfsPathParser(String parent) { + _parent = parent; + } + + @Override + public String interpret(BufferedReader reader) throws IOException { + String line; + while ((line = reader.readLine()) != null) { + paths.add(line); + } + return null; + } + + public List getPaths() { + return paths; + } + + @Override + public boolean drain() { + return true; + } + } + + @PreDestroy + public void destroy() { + s_logger.info("Clean up mounted NFS mount points used in current session."); + storageMounts.values().stream().forEach(this::umount); + } +} diff --git a/server/src/main/java/com/cloud/storage/secondary/CapacityChecker.java b/server/src/main/java/com/cloud/storage/secondary/CapacityChecker.java new file mode 100644 index 000000000000..2003fcbfecf7 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/secondary/CapacityChecker.java @@ -0,0 +1,18 @@ +package com.cloud.storage.secondary; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; + +public interface CapacityChecker { + double CAPACITY_THRESHOLD = 0.90; + + boolean hasEnoughCapacity(DataStore imageStore); + + default Long parse(String value) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + throw new CloudRuntimeException(String.format("unable to parse %s: %s.%n", value, e.getMessage())); + } + } +} diff --git a/server/src/main/java/com/cloud/storage/secondary/NfsMountCapacityChecker.java b/server/src/main/java/com/cloud/storage/secondary/NfsMountCapacityChecker.java new file mode 100644 index 000000000000..1bc731dcd3a5 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/secondary/NfsMountCapacityChecker.java @@ -0,0 +1,51 @@ +package com.cloud.storage.secondary; + +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.mount.MountManager; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Optional; + +@Component +public class NfsMountCapacityChecker implements CapacityChecker { + @Inject + private MountManager mountManager; + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; + + @Override + public boolean hasEnoughCapacity(DataStore imageStore) { + String[] capacityInfo = getCapacityInfo(imageStore); + Long capacity = parse(capacityInfo[0]); + Long used = parse(capacityInfo[1]); + if (used/(capacity * 1.0) <= CAPACITY_THRESHOLD) { + return true; + } + return false; + } + + private String[] getCapacityInfo(DataStore imageStore) { + String mountPoint = mountManager.getMountPoint(imageStore.getUri(), imageStoreDetailsUtil.getNfsVersion(imageStore.getId())); + String command = String.format("df %s | grep %s | awk -F' ' '{print $2, $3}'", mountPoint, hostFromUrl(imageStore.getUri())); + return Optional.ofNullable(Script.runSimpleBashScript(command)).orElse("0 0").split(" "); + } + + private String hostFromUrl(String url) { + try { + URI uri = new URI(UriUtils.encodeURIComponent(url)); + if (Optional.ofNullable(uri.getScheme()).orElse("").equalsIgnoreCase("nfs")) { + return uri.getHost(); + } + return url; + } catch (URISyntaxException e) { + throw new CloudRuntimeException(String.format("Invalid NFS url %s caused error: %s.%n", url, e.getMessage())); + } + } +} diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 8aa21661675f..814c7d174b51 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -16,38 +16,55 @@ // under the License. package com.cloud.template; +import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Config; +import com.cloud.configuration.Resource.ResourceType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.org.Grouping; import com.cloud.resource.ResourceManager; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ExecutionException; - -import javax.inject.Inject; - -import com.cloud.configuration.Config; +import com.cloud.server.StatsCollector; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.TemplateProfile; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.storage.download.DownloadMonitor; +import com.cloud.storage.secondary.CapacityChecker; +import com.cloud.template.VirtualMachineTemplate.State; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; -import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.log4j.Logger; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -61,37 +78,20 @@ import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.log4j.Logger; -import com.cloud.agent.AgentManager; -import com.cloud.alert.AlertManager; -import com.cloud.configuration.Resource.ResourceType; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.dao.DataCenterDao; -import com.cloud.event.EventTypes; -import com.cloud.event.UsageEventUtils; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.org.Grouping; -import com.cloud.server.StatsCollector; -import com.cloud.template.VirtualMachineTemplate.State; -import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.TemplateType; -import com.cloud.storage.TemplateProfile; -import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.VMTemplateZoneVO; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplateZoneDao; -import com.cloud.storage.download.DownloadMonitor; -import com.cloud.utils.UriUtils; -import com.cloud.utils.db.DB; -import com.cloud.utils.db.EntityManager; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; public class HypervisorTemplateAdapter extends TemplateAdapterBase { private final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); @@ -125,6 +125,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { ResourceManager resourceManager; @Inject VMTemplateDao templateDao; + @Inject + CapacityChecker capacityChecker; @Override public String getName() { @@ -171,6 +173,7 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio TemplateProfile profile = super.prepare(cmd); String url = profile.getUrl(); UriUtils.validateUrl(cmd.getFormat(), url); + UriUtils.checkUrlExistence(url); if (cmd.isDirectDownload()) { DigestHelper.validateChecksumString(cmd.getChecksum()); Long templateSize = performDirectDownloadUrlValidation(url); @@ -250,13 +253,14 @@ private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTempl } // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + if (Grouping.AllocationState.Disabled == zone.getAllocationState() && !template.getTemplateType().equals(Storage.TemplateType.SYSTEM)) { s_logger.info("Zone " + zoneId + " is disabled. Skip downloading template to its image store " + imageStore.getId()); continue; } // Check if image store has enough capacity for template - if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore) && + (template.getTemplateType().equals(Storage.TemplateType.SYSTEM) && !capacityChecker.hasEnoughCapacity(imageStore))) { s_logger.info("Image store doesn't have enough capacity. Skip downloading template to this image store " + imageStore.getId()); continue; } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index ebb73daa590d..3feb01a391c1 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -159,7 +159,9 @@ public TemplateProfile prepare(boolean isIso, long userId, String name, String d if (passwordEnabled == null) { passwordEnabled = false; } - if (requiresHVM == null) { + if (templateType == TemplateType.SYSTEM) { + requiresHVM = false; + } else if (requiresHVM == null) { requiresHVM = true; } } @@ -262,13 +264,12 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio Account caller = CallContext.current().getCallingAccount(); Account owner = _accountMgr.getAccount(cmd.getEntityOwnerId()); _accountMgr.checkAccess(caller, null, true, owner); - - boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); + TemplateType templateType = getTemplateType(cmd); List zoneId = cmd.getZoneIds(); - // ignore passed zoneId if we are using region wide image store + // ignore passed zoneId if we are using region wide image store or system template type List stores = _imgStoreDao.findRegionImageStores(); - if (stores != null && stores.size() > 0) { + if ((stores != null && stores.size() > 0) || templateType.equals(TemplateType.SYSTEM)){ zoneId = null; } @@ -280,8 +281,20 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(), cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true, - cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload()); + cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), templateType, cmd.isDirectDownload()); + + } + + private TemplateType getTemplateType(RegisterTemplateCmd cmd) { + boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); + if (cmd.isSystem()) { + return TemplateType.SYSTEM; + } else if (isRouting) { + return TemplateType.ROUTING; + } else { + return TemplateType.USER; + } } @Override diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index df3b59b5ff73..d34a8a3d1974 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -16,100 +16,6 @@ // under the License. package com.cloud.template; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import com.cloud.deploy.DeployDestination; -import com.cloud.storage.ImageStoreUploadMonitorImpl; -import com.cloud.utils.StringUtils; -import com.cloud.utils.EncryptionUtil; -import com.cloud.utils.DateUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.EnumUtils; -import com.google.common.base.Joiner; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; -import org.apache.cloudstack.framework.async.AsyncCallFuture; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.apache.log4j.Logger; -import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; -import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd; -import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; -import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; -import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; -import org.apache.cloudstack.api.command.user.iso.ListIsoPermissionsCmd; -import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; -import org.apache.cloudstack.api.command.user.iso.UpdateIsoCmd; -import org.apache.cloudstack.api.command.user.iso.UpdateIsoPermissionsCmd; -import org.apache.cloudstack.api.command.user.template.CopyTemplateCmd; -import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; -import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; -import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; -import org.apache.cloudstack.api.response.GetUploadParamsResponse; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.Scope; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.framework.messagebus.PublishScope; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CommandResult; -import org.apache.cloudstack.storage.command.DettachCommand; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; -import org.apache.cloudstack.storage.to.TemplateObjectTO; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -127,6 +33,7 @@ import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeployDestination; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; @@ -146,6 +53,7 @@ import com.cloud.projects.ProjectManager; import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; +import com.cloud.storage.ImageStoreUploadMonitorImpl; import com.cloud.storage.LaunchPermissionVO; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -177,6 +85,7 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.template.TemplateAdapter.TemplateAdapterType; import com.cloud.template.VirtualMachineTemplate.BootloaderType; +import com.cloud.upgrade.dao.VersionDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -184,6 +93,11 @@ import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.uservm.UserVm; +import com.cloud.utils.DateUtil; +import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.EnumUtils; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -200,10 +114,107 @@ import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; - +import com.google.common.base.Joiner; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; +import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd; +import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; +import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; +import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; +import org.apache.cloudstack.api.command.user.iso.ListIsoPermissionsCmd; +import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; +import org.apache.cloudstack.api.command.user.iso.UpdateIsoCmd; +import org.apache.cloudstack.api.command.user.iso.UpdateIsoPermissionsCmd; +import org.apache.cloudstack.api.command.user.template.CopyTemplateCmd; +import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; +import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; +import org.apache.cloudstack.api.response.GetUploadParamsResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.DettachCommand; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateHyperV; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateKvm; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateLxc; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateOvm3; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateVmware; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateXen; +import static com.cloud.storage.template.TemplateConstants.DEFAULT_BASE_SYSTEMVM_URL; + public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable { private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class); @@ -281,6 +292,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, MessageBus _messageBus; @Inject private VMTemplateDetailsDao _tmpltDetailsDao; + @Inject + private VersionDao _versionDao; private boolean _disableExtraction = false; private List _adapters; @@ -337,16 +350,21 @@ public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws U throw new PermissionDeniedException("Parameter isrouting can only be specified by a Root Admin, permission denied"); } } + if (cmd.isSystem()) { + if (!_accountService.isRootAdmin(account.getId())) { + throw new PermissionDeniedException(String.format("Value of '%s' for parameter %s can only be specified by a Root Admin, permission denied", + cmd.isSystem(), ApiConstants.SYSTEM)); + } + } TemplateAdapter adapter = getAdapter(HypervisorType.getType(cmd.getHypervisor())); TemplateProfile profile = adapter.prepare(cmd); - VMTemplateVO template = adapter.create(profile); + VMTemplateVO template = Optional.ofNullable(adapter.create(profile)).orElseThrow(() -> new CloudRuntimeException("Failed to create a template")); - if (template != null) { - return template; - } else { - throw new CloudRuntimeException("Failed to create a template"); + if (cmd.isSystem() && cmd.isActivate()) { + return activateSystemVMTemplate(template.getId()); } + return template; } @Override @@ -403,6 +421,107 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT } } + @Override + public VirtualMachineTemplate activateSystemVMTemplate(long templateId) { + VMTemplateVO template = Optional.ofNullable(_tmpltDao.findById(templateId)) + .orElseThrow(() -> new InvalidParameterValueException(String.format("Unable to find template with id %d.", templateId))); + Transaction.execute(new TransactionCallbackNoReturn() { + + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + updateTemplate(template); + updateVMInstances(template); + updateRouterTemplateConfig(template); + updateMinRequiredSystemVMVersionConfig(); + } + }); + + return template; + } + + private void updateTemplate(VMTemplateVO template) { + template.setTemplateType(TemplateType.SYSTEM); + _tmpltDao.update(template.getId(), template); + } + + private void updateVMInstances(VMTemplateVO template) { + List vmInstances = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); + vmInstances.stream().forEach(instance -> updateVmInstance(instance, template)); + } + + private void updateVmInstance(VMInstanceVO instance, VMTemplateVO template) { + instance.setTemplateId(template.getId()); + _vmInstanceDao.update(instance.getId(), instance); + } + + private void updateRouterTemplateConfig(VMTemplateVO template) { + String routerTemplateConfigKey = getRouterTemplateConfigKey(template.getHypervisorType()); + ConfigurationVO routerTemplate = Optional.ofNullable(_configDao.findByName(routerTemplateConfigKey)) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration for hypervisor %s: unable to find it.", routerTemplateConfigKey, template.getHypervisorType()))); + routerTemplate.setValue(template.getName()); + _configDao.update(routerTemplate.getName(), routerTemplate); + } + + private void updateMinRequiredSystemVMVersionConfig() { + String version = formatVersion(Optional.ofNullable(_versionDao.getCurrentVersion()) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find the current version.", NetworkOrchestrationService.MinVRVersion.key())))); + ConfigurationVO minRequiredVersion = Optional.ofNullable(_configDao.findByName(NetworkOrchestrationService.MinVRVersion.key())) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find it.", NetworkOrchestrationService.MinVRVersion.key()))); + minRequiredVersion.setValue(version); + _configDao.update(minRequiredVersion.getName(), minRequiredVersion); + } + + private String getRouterTemplateConfigKey(HypervisorType hypervisorType) { + switch (hypervisorType) { + case XenServer: + return RouterTemplateXen.key(); + case KVM: + return RouterTemplateKvm.key(); + case VMware: + return RouterTemplateVmware.key(); + case Hyperv: + return RouterTemplateHyperV.key(); + case LXC: + return RouterTemplateLxc.key(); + case Ovm3: + return RouterTemplateOvm3.key(); + default: + return ""; + } + } + + @Override + public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd) { + List vmTemplateVOS = getSystemVMTemplatesByUrlVersionMatch(cmd); + Optional maxIdTemplate = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); + return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), maxIdTemplate.orElse(vmTemplateVOS.get(0)).getUrl()); + } + + private List getSystemVMTemplatesByUrlVersionMatch(GetSystemVMTemplateDefaultURLCmd cmd) { + String version = resolveVersion(Optional.ofNullable(cmd.getVersion()).orElse("")); + List vmTemplateVOS = _tmpltDao.listSystemVMTemplatesByUrlLike(DEFAULT_BASE_SYSTEMVM_URL + "%" + version, cmd.getHypervisor()); + if (!vmTemplateVOS.isEmpty()) { + return vmTemplateVOS; + } else { + throw new CloudRuntimeException(String.format("Unable find System VM Template URL for version %s and hypervisor %s.", version, cmd.getHypervisor())); + } + } + + private String resolveVersion(String version) { + if (version.isEmpty()) { + version = Optional.ofNullable(_versionDao.getCurrentVersion()).orElse(""); + } + return formatVersion(version); + } + + private String formatVersion(String version) { + if (version.split("[.]").length > 3) { + return version.substring(0, org.apache.commons.lang3.StringUtils.ordinalIndexOf(version, ".", 3)); + } else { + return version; + } + } + @Override public DataStore getImageStore(String storeUuid, Long zoneId) { diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 2f67c4248d35..b1c809d2e3b2 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -65,6 +65,13 @@ + + + + + + + @@ -248,7 +255,9 @@ - + + +