From 68eeb0227d7fc7897e121ce46f32a04cf4b360be Mon Sep 17 00:00:00 2001 From: ernjvr Date: Thu, 11 Oct 2018 12:23:29 +0200 Subject: [PATCH 01/13] created new api skeletons and some implementation logic for it --- .../cloud/template/TemplateApiService.java | 7 ++ .../template/ActivateSystemVMTemplateCmd.java | 99 +++++++++++++++++++ .../GetSystemVMTemplateDefaultURLCmd.java | 93 +++++++++++++++++ ...GetSystemVMTemplateDefaultURLResponse.java | 46 +++++++++ .../com/cloud/storage/dao/VMTemplateDao.java | 2 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 16 +++ .../java/com/cloud/vm/dao/VMInstanceDao.java | 3 + .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 16 +++ ...spring-engine-schema-core-daos-context.xml | 1 + .../cloud/server/ManagementServerImpl.java | 4 + .../cloud/template/TemplateManagerImpl.java | 65 ++++++++++++ 11 files changed, 352 insertions(+) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 7348547cee05..b9c187a8d299 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -22,6 +22,8 @@ import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; +import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; @@ -39,6 +41,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; import org.apache.cloudstack.api.response.GetUploadParamsResponse; public interface TemplateApiService { @@ -47,6 +50,10 @@ public interface TemplateApiService { public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplateCmd cmd) throws ResourceAllocationException, MalformedURLException; + GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd); + + VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd); + VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java new file mode 100644 index 000000000000..764879e30c5a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.template; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.log4j.Logger; + +import java.util.List; + +@APICommand(name = "activateSystemVMTemplate", description = "Activates an existing system virtual machine template to be used by CloudStack to create system virtual machines.", + responseObject = TemplateResponse.class, authorized = {RoleType.Admin}) +public class ActivateSystemVMTemplateCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(ActivateSystemVMTemplateCmd.class.getName()); + private static final String NAME = "activatesystemvmtemplateresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = TemplateResponse.class, required = true, description = "The template ID of the System VM Template to activate.") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { +// try { + VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(this); + if (template != null) { + ListResponse response = new ListResponse(); + List templateResponses = _responseGenerator.createTemplateResponses(ResponseObject.ResponseView.Restricted, + template, 1L , false); + response.setResponses(templateResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to activate template."); + } +// } catch ( ex) { +// LOGGER.warn("Exception: ", ex); +// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); +// } + } + + @Override + public String getCommandName() { + return NAME; + } + + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Template; + } + + @Override + public long getEntityOwnerId() { + final VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, id); + if (template != null) { + return template.getAccountId(); + } + + // bad id given, parent this command to SYSTEM so ERROR events are tracked + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java new file mode 100644 index 000000000000..7638dd0e94b2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.template; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "getSystemVMTemplateDefaultURL", description = "Gets the system virtual machine template's default download URL.", + responseObject = GetSystemVMTemplateDefaultURLResponse.class, authorized = {RoleType.Admin}) +public class GetSystemVMTemplateDefaultURLCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(GetSystemVMTemplateDefaultURLCmd.class.getName()); + private static final String NAME = "getsystemvmtemplatedefaulturlresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, + entityType = TemplateResponse.class, required = false, description = "The CloudStack version for which to get System VM Template URL.") + private String version; + + @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, entityType = TemplateResponse.class, required = true, description = "The hypervisor for which to get System VM Template URL.") + private String hypervisor; + + public String getVersion() { + return version; + } + + public String getHypervisor() { + return hypervisor; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { +// VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, 1); +// String url = template.getUrl(); +// if (url != null && !url.isEmpty()) { +// setResponseObject(new GetSystemVMTemplateDefaultURLResponse(getCommandName(), url)); +// } else { +// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to find URL for version '%s' and hypervisor '%s'", version, hypervisor)); +// } + GetSystemVMTemplateDefaultURLResponse response = _templateService.getSystemVMTemplateDefaultURL(this); + if (response != null) { + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to find URL for version '%s' and hypervisor '%s'", version, hypervisor)); + } + } + + @Override + public String getCommandName() { + return NAME; + } + + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Template; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java new file mode 100644 index 000000000000..272bfdfb8605 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +public class GetSystemVMTemplateDefaultURLResponse extends BaseResponse { + + @SerializedName(ApiConstants.URL) + @Param(description = "The default URL of the System VM template.") + private String url; + + public GetSystemVMTemplateDefaultURLResponse() { + } + + public GetSystemVMTemplateDefaultURLResponse(String responseName, String url) { + setResponseName(responseName); + setObjectName("url"); + this.url = url; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index c43a2ea4ee9e..a0296ba23478 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -40,6 +40,8 @@ public interface VMTemplateDao extends GenericDao, StateDao< public List listAllSystemVMTemplates(); + List listSystemVMTemplatesByUrlLike(String partialUrl, String hType); + public List listDefaultBuiltinTemplates(); public String getRoutingTemplateUniqueName(); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index dd1f2fcf1641..8a6d6398b8d6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -92,6 +92,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem protected SearchBuilder tmpltTypeHyperSearch; protected SearchBuilder readySystemTemplateSearch; protected SearchBuilder tmpltTypeHyperSearch2; + protected SearchBuilder urlLikeSearch; protected SearchBuilder AccountIdSearch; protected SearchBuilder NameSearch; @@ -221,6 +222,15 @@ public List listAllSystemVMTemplates() { return listBy(sc, filter); } + @Override + public List listSystemVMTemplatesByUrlLike(String partialUrl, String hypervisorType) { + SearchCriteria sc = urlLikeSearch.create(); + sc.setParameters("templateType", TemplateType.SYSTEM); + sc.setParameters("hypervisorType", hypervisorType); + sc.setParameters("url", "%" + partialUrl + "%"); + return listBy(sc); + } + @Override public List listPrivateTemplatesByHost(Long hostId) { @@ -417,6 +427,12 @@ public boolean configure(String name, Map params) throws Configu ParentTemplateIdSearch.and("state", ParentTemplateIdSearch.entity().getState(), SearchCriteria.Op.EQ); ParentTemplateIdSearch.done(); + urlLikeSearch = createSearchBuilder(); + urlLikeSearch.and("templateType", urlLikeSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); + urlLikeSearch.and("hypervisorType", urlLikeSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + urlLikeSearch.and("url", urlLikeSearch.entity().getUrl(), SearchCriteria.Op.LIKE); + urlLikeSearch.done(); + return result; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 6fda4a15c324..47c666b0b8ff 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; @@ -82,6 +83,8 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listByTypes(VirtualMachine.Type... types); + List listByHypervisorTypeAndNonUserTypes(HypervisorType hypervisorType); + VMInstanceVO findByIdTypes(long id, VirtualMachine.Type... types); VMInstanceVO findVMByInstanceName(String name); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 1565f53233bc..1d5ae7880071 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -28,6 +28,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -77,6 +78,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder StateChangeSearch; protected SearchBuilder TransitionSearch; protected SearchBuilder TypesSearch; + protected SearchBuilder hypervisorTypeAndNonUserTypeSearch; protected SearchBuilder IdTypesSearch; protected SearchBuilder HostIdTypesSearch; protected SearchBuilder HostIdStatesSearch; @@ -193,6 +195,12 @@ protected void init() { TypesSearch.and("types", TypesSearch.entity().getType(), Op.IN); TypesSearch.done(); + hypervisorTypeAndNonUserTypeSearch = createSearchBuilder(); + hypervisorTypeAndNonUserTypeSearch.and("hypervisorType", + hypervisorTypeAndNonUserTypeSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + hypervisorTypeAndNonUserTypeSearch.and("types", hypervisorTypeAndNonUserTypeSearch.entity().getType(), Op.NIN); + hypervisorTypeAndNonUserTypeSearch.done(); + IdTypesSearch = createSearchBuilder(); IdTypesSearch.and("id", IdTypesSearch.entity().getId(), Op.EQ); IdTypesSearch.and("types", IdTypesSearch.entity().getType(), Op.IN); @@ -427,6 +435,14 @@ public List listByTypes(Type... types) { return listBy(sc); } + @Override + public List listByHypervisorTypeAndNonUserTypes(HypervisorType hypervisorType) { + SearchCriteria sc = hypervisorTypeAndNonUserTypeSearch.create(); + sc.setParameters("hypervisorType", hypervisorType); + sc.setParameters("types", Type.User); + return listBy(sc); + } + @Override public List listByTypeAndState(VirtualMachine.Type type, State state) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 84c27583925b..6238484deef6 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -356,4 +356,5 @@ + diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 4cd2811cc204..a432744a8332 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -206,10 +206,12 @@ import org.apache.cloudstack.api.command.admin.systemvm.UpgradeSystemVMCmd; import org.apache.cloudstack.api.command.admin.template.CopyTemplateCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.CreateTemplateCmdByAdmin; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.admin.template.ListTemplatePermissionsCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.ListTemplatesCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.PrepareTemplateCmd; import org.apache.cloudstack.api.command.admin.template.RegisterTemplateCmdByAdmin; +import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; import org.apache.cloudstack.api.command.admin.usage.AddTrafficMonitorCmd; import org.apache.cloudstack.api.command.admin.usage.AddTrafficTypeCmd; import org.apache.cloudstack.api.command.admin.usage.DeleteTrafficMonitorCmd; @@ -2842,6 +2844,8 @@ public List> getCommands() { cmdList.add(ListTemplatePermissionsCmd.class); cmdList.add(ListTemplatesCmd.class); cmdList.add(RegisterTemplateCmd.class); + cmdList.add(ActivateSystemVMTemplateCmd.class); + cmdList.add(GetSystemVMTemplateDefaultURLCmd.class); cmdList.add(UpdateTemplateCmd.class); cmdList.add(UpdateTemplatePermissionsCmd.class); cmdList.add(AddNicToVMCmd.class); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index dc57a8a9ef36..a5ce5a2072cc 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -20,10 +20,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -34,6 +36,7 @@ import com.cloud.deploy.DeployDestination; import com.cloud.storage.ImageStoreUploadMonitorImpl; +import com.cloud.upgrade.dao.VersionDao; import com.cloud.utils.StringUtils; import com.cloud.utils.EncryptionUtil; import com.cloud.utils.DateUtil; @@ -43,7 +46,10 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; @@ -280,6 +286,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, MessageBus _messageBus; @Inject private VMTemplateDetailsDao _tmpltDetailsDao; + @Inject + private VersionDao _versionDao; private boolean _disableExtraction = false; private List _adapters; @@ -291,6 +299,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject private EndPointSelector selector; + public static final String BASE_SYSTEMVM_URL = "https://download.cloudstack.org/systemvm"; + private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; @@ -402,6 +412,61 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT } } + @Override + public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd) { + VMTemplateVO template = _tmpltDao.findById(cmd.getId()); + if (template == null) { + throw new InvalidParameterValueException(String.format("Unable to find template with id %d" + cmd.getId())); + } + Transaction.execute(new TransactionCallbackNoReturn() { + + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + //Update the vm_template type field to ‘SYSTEM’ in database for the given id. + template.setTemplateType(TemplateType.SYSTEM); + boolean update = _tmpltDao.update(cmd.getId(), template); + //o Update the vm_instance records with the template ID for the specified hypervisor type in the database. + List vmInstanceVOS = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); + //o Update the router.template global configuration parameter in the database. + // + //o Update minreq.systemtemplate.version configuration parameter in the database. + } + + }); + + return template; + } + + @Override + public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd) { + String version = resolveVersion(cmd.getVersion()); + + List vmTemplateVOS = _tmpltDao.listSystemVMTemplatesByUrlLike(BASE_SYSTEMVM_URL + "%" + version, cmd.getHypervisor()); + if (!vmTemplateVOS.isEmpty()) { + Optional max = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); + return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), max.orElse(vmTemplateVOS.get(0)).getUrl()); + } else { + throw new CloudRuntimeException(String.format("cannot find System VM Template URL for version %s", version)); + } + } + + private String resolveVersion(String version) { + if (version == null || version.isEmpty()) { + version = _versionDao.getCurrentVersion(); + } + return formatVersion(version); + } + + private String formatVersion(String version) { + if (version == null) { + return ""; + } else if (version.split("[.]").length > 3) { + return version.substring(0, org.apache.commons.lang3.StringUtils.ordinalIndexOf(version, ".", 3)); + } else { + return version; + } + } + @Override public DataStore getImageStore(String storeUuid, Long zoneId) { From cff84fac9c3be2bca0012bec55bb4bd6435f7871 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Fri, 12 Oct 2018 11:00:48 +0200 Subject: [PATCH 02/13] activateSystemVMTemplate api implementation logic --- .../cloud/template/TemplateManagerImpl.java | 69 ++++++++++++++++--- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index a5ce5a2072cc..294c47d3652f 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -50,7 +50,9 @@ import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; @@ -209,6 +211,13 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateHyperV; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateKvm; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateLxc; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateOvm3; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateVmware; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateXen; + public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable { private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class); @@ -416,7 +425,7 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd) { VMTemplateVO template = _tmpltDao.findById(cmd.getId()); if (template == null) { - throw new InvalidParameterValueException(String.format("Unable to find template with id %d" + cmd.getId())); + throw new InvalidParameterValueException(String.format("Unable to find template with id %d.", cmd.getId())); } Transaction.execute(new TransactionCallbackNoReturn() { @@ -424,19 +433,61 @@ public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateC public void doInTransactionWithoutResult(TransactionStatus status) { //Update the vm_template type field to ‘SYSTEM’ in database for the given id. template.setTemplateType(TemplateType.SYSTEM); - boolean update = _tmpltDao.update(cmd.getId(), template); - //o Update the vm_instance records with the template ID for the specified hypervisor type in the database. - List vmInstanceVOS = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); - //o Update the router.template global configuration parameter in the database. - // - //o Update minreq.systemtemplate.version configuration parameter in the database. - } + _tmpltDao.update(template.getId(), template); + + //Update the vm_instance records with the template ID for the specified hypervisor type in the database. + List vmInstances = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); + vmInstances.stream().forEach(instance -> { + instance.setTemplateId(template.getId()); + _vmInstanceDao.update(instance.getId(), instance); + }); + + //Update the router.template.x global configuration parameter in the database. + ConfigurationVO routerTemplate = _configDao.findByName(getRouterTemplateConfigKey(template.getHypervisorType())); + if (routerTemplate != null) { + routerTemplate.setValue(template.getName()); + _configDao.update(routerTemplate.getName(), routerTemplate); + } else { + throw new CloudRuntimeException(String.format("Cannot update router template configuration for hypervisor %s: unable to find it.", template.getHypervisorType())); + } + //Update minreq.systemtemplate.version configuration parameter in the database. + String version = formatVersion(_versionDao.getCurrentVersion()); + if (version.isEmpty()) { + throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find the current version.", NetworkOrchestrationService.MinVRVersion.key())); + } + ConfigurationVO minRequiredVersion = _configDao.findByName(NetworkOrchestrationService.MinVRVersion.key()); + if (minRequiredVersion != null) { + minRequiredVersion.setValue(version); + _configDao.update(minRequiredVersion.getName(), minRequiredVersion); + } else { + throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find it.", NetworkOrchestrationService.MinVRVersion.key())); + } + } }); return template; } + private String getRouterTemplateConfigKey(HypervisorType hypervisorType) { + switch (hypervisorType) { + case XenServer: + return RouterTemplateXen.key(); + case KVM: + return RouterTemplateKvm.key(); + case VMware: + return RouterTemplateVmware.key(); + case Hyperv: + return RouterTemplateHyperV.key(); + case LXC: + return RouterTemplateLxc.key(); + case Ovm3: + return RouterTemplateOvm3.key(); + default: + return ""; + } + } + @Override public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd) { String version = resolveVersion(cmd.getVersion()); @@ -446,7 +497,7 @@ public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSy Optional max = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), max.orElse(vmTemplateVOS.get(0)).getUrl()); } else { - throw new CloudRuntimeException(String.format("cannot find System VM Template URL for version %s", version)); + throw new CloudRuntimeException(String.format("Unable find System VM Template URL for version %s", version)); } } From ebb4d0916ae34a7bd9807903d509f81941ab7975 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Fri, 12 Oct 2018 11:41:58 +0200 Subject: [PATCH 03/13] clean up --- .../admin/template/ActivateSystemVMTemplateCmd.java | 5 ----- .../admin/template/GetSystemVMTemplateDefaultURLCmd.java | 7 ------- 2 files changed, 12 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java index 764879e30c5a..259351cf97c8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java @@ -59,7 +59,6 @@ public Long getId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { -// try { VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(this); if (template != null) { ListResponse response = new ListResponse(); @@ -71,10 +70,6 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to activate template."); } -// } catch ( ex) { -// LOGGER.warn("Exception: ", ex); -// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); -// } } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java index 7638dd0e94b2..2057f5278332 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java @@ -62,13 +62,6 @@ public String getHypervisor() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { -// VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, 1); -// String url = template.getUrl(); -// if (url != null && !url.isEmpty()) { -// setResponseObject(new GetSystemVMTemplateDefaultURLResponse(getCommandName(), url)); -// } else { -// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to find URL for version '%s' and hypervisor '%s'", version, hypervisor)); -// } GetSystemVMTemplateDefaultURLResponse response = _templateService.getSystemVMTemplateDefaultURL(this); if (response != null) { setResponseObject(response); From 932b217af08e2f502a8cd4094d1f5b95cb5e0539 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 17 Oct 2018 17:28:45 +0200 Subject: [PATCH 04/13] enhance register system vm api & refactor other --- .../cloud/template/TemplateApiService.java | 3 +- .../apache/cloudstack/api/ApiConstants.java | 2 + .../template/ActivateSystemVMTemplateCmd.java | 2 +- .../user/template/RegisterTemplateCmd.java | 25 +++- .../storage/template/TemplateConstants.java | 2 +- .../template/HypervisorTemplateAdapter.java | 1 + .../cloud/template/TemplateAdapterBase.java | 25 +++- .../cloud/template/TemplateManagerImpl.java | 118 ++++++++++-------- 8 files changed, 112 insertions(+), 66 deletions(-) diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index b9c187a8d299..16d61bf89a13 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -22,7 +22,6 @@ import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; -import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; @@ -52,7 +51,7 @@ public interface TemplateApiService { GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd); - VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd); + VirtualMachineTemplate activateSystemVMTemplate(long templateId); VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index b7779cb2a8d9..34d30cd803b3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -299,6 +299,7 @@ public class ApiConstants { public static final String TARGET_IQN = "targetiqn"; public static final String TEMPLATE_FILTER = "templatefilter"; public static final String TEMPLATE_ID = "templateid"; + public static final String TEMPLATE_TYPE = "templatetype"; public static final String ISO_ID = "isoid"; public static final String TIMEOUT = "timeout"; public static final String TIMEZONE = "timezone"; @@ -602,6 +603,7 @@ public class ApiConstants { public static final String INTERVAL = "interval"; public static final String QUIETTIME = "quiettime"; public static final String ACTION = "action"; + public static final String ACTIVATE = "activate"; public static final String CONDITION_ID = "conditionid"; public static final String CONDITION_IDS = "conditionids"; public static final String COUNTERPARAM_LIST = "counterparam"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java index 259351cf97c8..36097d67b9f2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java @@ -59,7 +59,7 @@ public Long getId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(this); + VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(id); if (template != null) { ListResponse response = new ListResponse(); List templateResponses = _responseGenerator.createTemplateResponses(ResponseObject.ResponseView.Restricted, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 333b363d16a9..6cd6133a11d3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -22,7 +22,9 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; +import com.cloud.storage.Storage.TemplateType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; import org.apache.cloudstack.api.ApiConstants; @@ -161,6 +163,13 @@ public class RegisterTemplateCmd extends BaseCmd { description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment") private Boolean directDownload; + @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING, description = "the template type. Possible values include user and system.") + private String templateType; + + @Parameter(name=ApiConstants.ACTIVATE, type = CommandType.BOOLEAN, + description = "true if this template should be used by CloudStack to create System VMs. Must be used with templatetype of 'system'.") + private Boolean activate; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -273,6 +282,14 @@ public boolean isDirectDownload() { return directDownload == null ? false : directDownload; } + public String getTemplateType() { + return Optional.ofNullable(templateType).orElse(""); + } + + public Boolean isActivate() { + return Optional.ofNullable(activate).orElse(false); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -319,11 +336,17 @@ public void execute() throws ResourceAllocationException { } protected void validateParameters() { + if ((!getTemplateType().isEmpty() && !getTemplateType().equalsIgnoreCase(TemplateType.USER.name()) + && !getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name()))) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Invalid parameter value specified for templateType. Valid values: user or system"); + } if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty())) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Both zoneid and zoneids cannot be specified at the same time"); - if (zoneId == null && (zoneIds == null || zoneIds.isEmpty())) + if ((zoneId == null && (zoneIds == null || zoneIds.isEmpty())) && + (!getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name()))) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Either zoneid or zoneids is required. Both cannot be null."); diff --git a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java index 25c2d5b3c074..55be0b2c409d 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java @@ -26,7 +26,7 @@ public final class TemplateConstants { public static final String DEFAULT_TMPLT_FIRST_LEVEL_DIR = "tmpl/"; public static final String DEFAULT_SYSTEM_VM_TEMPLATE_PATH = "template/tmpl/1/"; - + public static final String DEFAULT_BASE_SYSTEMVM_URL = "https://download.cloudstack.org/systemvm"; public static final String DEFAULT_SYSTEM_VM_TMPLT_NAME = "routing"; public static final int DEFAULT_TMPLT_COPY_PORT = 80; diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index bfa73af6bcd3..15fedb2057a4 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -169,6 +169,7 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio TemplateProfile profile = super.prepare(cmd); String url = profile.getUrl(); UriUtils.validateUrl(cmd.getFormat(), url); + UriUtils.checkUrlExistence(url); if (cmd.isDirectDownload()) { Long templateSize = performDirectDownloadUrlValidation(url); profile.setSize(templateSize); diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index ebb73daa590d..a5830552a59b 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -159,7 +159,9 @@ public TemplateProfile prepare(boolean isIso, long userId, String name, String d if (passwordEnabled == null) { passwordEnabled = false; } - if (requiresHVM == null) { + if (templateType == TemplateType.SYSTEM) { + requiresHVM = false; + } else if (requiresHVM == null) { requiresHVM = true; } } @@ -262,13 +264,12 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio Account caller = CallContext.current().getCallingAccount(); Account owner = _accountMgr.getAccount(cmd.getEntityOwnerId()); _accountMgr.checkAccess(caller, null, true, owner); - - boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); + TemplateType templateType = getTemplateType(cmd); List zoneId = cmd.getZoneIds(); - // ignore passed zoneId if we are using region wide image store + // ignore passed zoneId if we are using region wide image store or system template type List stores = _imgStoreDao.findRegionImageStores(); - if (stores != null && stores.size() > 0) { + if ((stores != null && stores.size() > 0) || templateType.equals(TemplateType.SYSTEM)){ zoneId = null; } @@ -280,8 +281,20 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(), cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true, - cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload()); + cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), templateType, cmd.isDirectDownload()); + + } + + private TemplateType getTemplateType(RegisterTemplateCmd cmd) { + boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); + if (cmd.getTemplateType() != null && cmd.getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name())) { + return TemplateType.SYSTEM; + } else if (isRouting) { + return TemplateType.ROUTING; + } else { + return TemplateType.USER; + } } @Override diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 294c47d3652f..ca74b552de79 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -46,7 +46,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; @@ -217,6 +216,7 @@ import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateOvm3; import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateVmware; import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateXen; +import static com.cloud.storage.template.TemplateConstants.DEFAULT_BASE_SYSTEMVM_URL; public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable { private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class); @@ -308,8 +308,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject private EndPointSelector selector; - public static final String BASE_SYSTEMVM_URL = "https://download.cloudstack.org/systemvm"; - private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; @@ -355,16 +353,22 @@ public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws U throw new PermissionDeniedException("Parameter isrouting can only be specified by a Root Admin, permission denied"); } } + if (cmd.getTemplateType().equalsIgnoreCase(Storage.TemplateType.SYSTEM.name())) { + if (!_accountService.isRootAdmin(account.getId())) { + throw new PermissionDeniedException(String.format("Value of '%s' for parameter templatetype can only be specified by a Root Admin, permission denied", + Storage.TemplateType.SYSTEM.name())); + } + } TemplateAdapter adapter = getAdapter(HypervisorType.getType(cmd.getHypervisor())); TemplateProfile profile = adapter.prepare(cmd); - VMTemplateVO template = adapter.create(profile); + VMTemplateVO template = Optional.ofNullable(adapter.create(profile)).orElseThrow(() -> new CloudRuntimeException("Failed to create a template")); - if (template != null) { - return template; - } else { - throw new CloudRuntimeException("Failed to create a template"); + if (cmd.getTemplateType().equalsIgnoreCase(Storage.TemplateType.SYSTEM.name()) + && cmd.isActivate()) { + return activateSystemVMTemplate(template.getId()); } + return template; } @Override @@ -422,53 +426,55 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT } @Override - public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd) { - VMTemplateVO template = _tmpltDao.findById(cmd.getId()); - if (template == null) { - throw new InvalidParameterValueException(String.format("Unable to find template with id %d.", cmd.getId())); - } + public VirtualMachineTemplate activateSystemVMTemplate(long templateId) { + VMTemplateVO template = Optional.ofNullable(_tmpltDao.findById(templateId)) + .orElseThrow(() -> new InvalidParameterValueException(String.format("Unable to find template with id %d.", templateId))); Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - //Update the vm_template type field to ‘SYSTEM’ in database for the given id. - template.setTemplateType(TemplateType.SYSTEM); - _tmpltDao.update(template.getId(), template); - - //Update the vm_instance records with the template ID for the specified hypervisor type in the database. - List vmInstances = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); - vmInstances.stream().forEach(instance -> { - instance.setTemplateId(template.getId()); - _vmInstanceDao.update(instance.getId(), instance); - }); - - //Update the router.template.x global configuration parameter in the database. - ConfigurationVO routerTemplate = _configDao.findByName(getRouterTemplateConfigKey(template.getHypervisorType())); - if (routerTemplate != null) { - routerTemplate.setValue(template.getName()); - _configDao.update(routerTemplate.getName(), routerTemplate); - } else { - throw new CloudRuntimeException(String.format("Cannot update router template configuration for hypervisor %s: unable to find it.", template.getHypervisorType())); - } - - //Update minreq.systemtemplate.version configuration parameter in the database. - String version = formatVersion(_versionDao.getCurrentVersion()); - if (version.isEmpty()) { - throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find the current version.", NetworkOrchestrationService.MinVRVersion.key())); - } - ConfigurationVO minRequiredVersion = _configDao.findByName(NetworkOrchestrationService.MinVRVersion.key()); - if (minRequiredVersion != null) { - minRequiredVersion.setValue(version); - _configDao.update(minRequiredVersion.getName(), minRequiredVersion); - } else { - throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find it.", NetworkOrchestrationService.MinVRVersion.key())); - } + updateTemplate(template); + updateVMInstances(template); + updateRouterTemplateConfig(template); + updateMinRequiredSystemVMVersionConfig(); } }); return template; } + private void updateTemplate(VMTemplateVO template) { + template.setTemplateType(TemplateType.SYSTEM); + _tmpltDao.update(template.getId(), template); + } + + private void updateVMInstances(VMTemplateVO template) { + List vmInstances = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); + vmInstances.stream().forEach(instance -> updateVmInstance(instance, template)); + } + + private void updateVmInstance(VMInstanceVO instance, VMTemplateVO template) { + instance.setTemplateId(template.getId()); + _vmInstanceDao.update(instance.getId(), instance); + } + + private void updateRouterTemplateConfig(VMTemplateVO template) { + String routerTemplateConfigKey = getRouterTemplateConfigKey(template.getHypervisorType()); + ConfigurationVO routerTemplate = Optional.ofNullable(_configDao.findByName(routerTemplateConfigKey)) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration for hypervisor %s: unable to find it.", routerTemplateConfigKey, template.getHypervisorType()))); + routerTemplate.setValue(template.getName()); + _configDao.update(routerTemplate.getName(), routerTemplate); + } + + private void updateMinRequiredSystemVMVersionConfig() { + String version = formatVersion(Optional.ofNullable(_versionDao.getCurrentVersion()) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find the current version.", NetworkOrchestrationService.MinVRVersion.key())))); + ConfigurationVO minRequiredVersion = Optional.ofNullable(_configDao.findByName(NetworkOrchestrationService.MinVRVersion.key())) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find it.", NetworkOrchestrationService.MinVRVersion.key()))); + minRequiredVersion.setValue(version); + _configDao.update(minRequiredVersion.getName(), minRequiredVersion); + } + private String getRouterTemplateConfigKey(HypervisorType hypervisorType) { switch (hypervisorType) { case XenServer: @@ -490,28 +496,30 @@ private String getRouterTemplateConfigKey(HypervisorType hypervisorType) { @Override public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd) { - String version = resolveVersion(cmd.getVersion()); + List vmTemplateVOS = getSystemVMTemplatesByUrlVersionMatch(cmd); + Optional maxIdTemplate = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); + return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), maxIdTemplate.orElse(vmTemplateVOS.get(0)).getUrl()); + } - List vmTemplateVOS = _tmpltDao.listSystemVMTemplatesByUrlLike(BASE_SYSTEMVM_URL + "%" + version, cmd.getHypervisor()); + private List getSystemVMTemplatesByUrlVersionMatch(GetSystemVMTemplateDefaultURLCmd cmd) { + String version = resolveVersion(Optional.ofNullable(cmd.getVersion()).orElse("")); + List vmTemplateVOS = _tmpltDao.listSystemVMTemplatesByUrlLike(DEFAULT_BASE_SYSTEMVM_URL + "%" + version, cmd.getHypervisor()); if (!vmTemplateVOS.isEmpty()) { - Optional max = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); - return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), max.orElse(vmTemplateVOS.get(0)).getUrl()); + return vmTemplateVOS; } else { - throw new CloudRuntimeException(String.format("Unable find System VM Template URL for version %s", version)); + throw new CloudRuntimeException(String.format("Unable find System VM Template URL for version %s and hypervisor %s.", version, cmd.getHypervisor())); } } private String resolveVersion(String version) { - if (version == null || version.isEmpty()) { - version = _versionDao.getCurrentVersion(); + if (version.isEmpty()) { + version = Optional.ofNullable(_versionDao.getCurrentVersion()).orElse(""); } return formatVersion(version); } private String formatVersion(String version) { - if (version == null) { - return ""; - } else if (version.split("[.]").length > 3) { + if (version.split("[.]").length > 3) { return version.substring(0, org.apache.commons.lang3.StringUtils.ordinalIndexOf(version, ".", 3)); } else { return version; From 2d9bff38a522ecb9dbf3244fd5af5ab60cac4c00 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Thu, 11 Oct 2018 12:23:29 +0200 Subject: [PATCH 05/13] created new api skeletons and some implementation logic for it --- .../cloud/template/TemplateApiService.java | 7 ++ .../template/ActivateSystemVMTemplateCmd.java | 99 +++++++++++++++++++ .../GetSystemVMTemplateDefaultURLCmd.java | 93 +++++++++++++++++ ...GetSystemVMTemplateDefaultURLResponse.java | 46 +++++++++ .../com/cloud/storage/dao/VMTemplateDao.java | 2 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 16 +++ .../java/com/cloud/vm/dao/VMInstanceDao.java | 3 + .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 16 +++ ...spring-engine-schema-core-daos-context.xml | 1 + .../cloud/server/ManagementServerImpl.java | 4 + .../cloud/template/TemplateManagerImpl.java | 65 ++++++++++++ 11 files changed, 352 insertions(+) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 7348547cee05..b9c187a8d299 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -22,6 +22,8 @@ import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; +import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; @@ -39,6 +41,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; import org.apache.cloudstack.api.response.GetUploadParamsResponse; public interface TemplateApiService { @@ -47,6 +50,10 @@ public interface TemplateApiService { public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplateCmd cmd) throws ResourceAllocationException, MalformedURLException; + GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd); + + VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd); + VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java new file mode 100644 index 000000000000..764879e30c5a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.template; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.log4j.Logger; + +import java.util.List; + +@APICommand(name = "activateSystemVMTemplate", description = "Activates an existing system virtual machine template to be used by CloudStack to create system virtual machines.", + responseObject = TemplateResponse.class, authorized = {RoleType.Admin}) +public class ActivateSystemVMTemplateCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(ActivateSystemVMTemplateCmd.class.getName()); + private static final String NAME = "activatesystemvmtemplateresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = TemplateResponse.class, required = true, description = "The template ID of the System VM Template to activate.") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { +// try { + VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(this); + if (template != null) { + ListResponse response = new ListResponse(); + List templateResponses = _responseGenerator.createTemplateResponses(ResponseObject.ResponseView.Restricted, + template, 1L , false); + response.setResponses(templateResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to activate template."); + } +// } catch ( ex) { +// LOGGER.warn("Exception: ", ex); +// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); +// } + } + + @Override + public String getCommandName() { + return NAME; + } + + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Template; + } + + @Override + public long getEntityOwnerId() { + final VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, id); + if (template != null) { + return template.getAccountId(); + } + + // bad id given, parent this command to SYSTEM so ERROR events are tracked + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java new file mode 100644 index 000000000000..7638dd0e94b2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.template; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "getSystemVMTemplateDefaultURL", description = "Gets the system virtual machine template's default download URL.", + responseObject = GetSystemVMTemplateDefaultURLResponse.class, authorized = {RoleType.Admin}) +public class GetSystemVMTemplateDefaultURLCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(GetSystemVMTemplateDefaultURLCmd.class.getName()); + private static final String NAME = "getsystemvmtemplatedefaulturlresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, + entityType = TemplateResponse.class, required = false, description = "The CloudStack version for which to get System VM Template URL.") + private String version; + + @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, entityType = TemplateResponse.class, required = true, description = "The hypervisor for which to get System VM Template URL.") + private String hypervisor; + + public String getVersion() { + return version; + } + + public String getHypervisor() { + return hypervisor; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { +// VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, 1); +// String url = template.getUrl(); +// if (url != null && !url.isEmpty()) { +// setResponseObject(new GetSystemVMTemplateDefaultURLResponse(getCommandName(), url)); +// } else { +// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to find URL for version '%s' and hypervisor '%s'", version, hypervisor)); +// } + GetSystemVMTemplateDefaultURLResponse response = _templateService.getSystemVMTemplateDefaultURL(this); + if (response != null) { + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to find URL for version '%s' and hypervisor '%s'", version, hypervisor)); + } + } + + @Override + public String getCommandName() { + return NAME; + } + + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.Template; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java new file mode 100644 index 000000000000..272bfdfb8605 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/GetSystemVMTemplateDefaultURLResponse.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +public class GetSystemVMTemplateDefaultURLResponse extends BaseResponse { + + @SerializedName(ApiConstants.URL) + @Param(description = "The default URL of the System VM template.") + private String url; + + public GetSystemVMTemplateDefaultURLResponse() { + } + + public GetSystemVMTemplateDefaultURLResponse(String responseName, String url) { + setResponseName(responseName); + setObjectName("url"); + this.url = url; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index c43a2ea4ee9e..a0296ba23478 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -40,6 +40,8 @@ public interface VMTemplateDao extends GenericDao, StateDao< public List listAllSystemVMTemplates(); + List listSystemVMTemplatesByUrlLike(String partialUrl, String hType); + public List listDefaultBuiltinTemplates(); public String getRoutingTemplateUniqueName(); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index dd1f2fcf1641..8a6d6398b8d6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -92,6 +92,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem protected SearchBuilder tmpltTypeHyperSearch; protected SearchBuilder readySystemTemplateSearch; protected SearchBuilder tmpltTypeHyperSearch2; + protected SearchBuilder urlLikeSearch; protected SearchBuilder AccountIdSearch; protected SearchBuilder NameSearch; @@ -221,6 +222,15 @@ public List listAllSystemVMTemplates() { return listBy(sc, filter); } + @Override + public List listSystemVMTemplatesByUrlLike(String partialUrl, String hypervisorType) { + SearchCriteria sc = urlLikeSearch.create(); + sc.setParameters("templateType", TemplateType.SYSTEM); + sc.setParameters("hypervisorType", hypervisorType); + sc.setParameters("url", "%" + partialUrl + "%"); + return listBy(sc); + } + @Override public List listPrivateTemplatesByHost(Long hostId) { @@ -417,6 +427,12 @@ public boolean configure(String name, Map params) throws Configu ParentTemplateIdSearch.and("state", ParentTemplateIdSearch.entity().getState(), SearchCriteria.Op.EQ); ParentTemplateIdSearch.done(); + urlLikeSearch = createSearchBuilder(); + urlLikeSearch.and("templateType", urlLikeSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); + urlLikeSearch.and("hypervisorType", urlLikeSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + urlLikeSearch.and("url", urlLikeSearch.entity().getUrl(), SearchCriteria.Op.LIKE); + urlLikeSearch.done(); + return result; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 6fda4a15c324..47c666b0b8ff 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; @@ -82,6 +83,8 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listByTypes(VirtualMachine.Type... types); + List listByHypervisorTypeAndNonUserTypes(HypervisorType hypervisorType); + VMInstanceVO findByIdTypes(long id, VirtualMachine.Type... types); VMInstanceVO findVMByInstanceName(String name); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 1565f53233bc..1d5ae7880071 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -28,6 +28,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -77,6 +78,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder StateChangeSearch; protected SearchBuilder TransitionSearch; protected SearchBuilder TypesSearch; + protected SearchBuilder hypervisorTypeAndNonUserTypeSearch; protected SearchBuilder IdTypesSearch; protected SearchBuilder HostIdTypesSearch; protected SearchBuilder HostIdStatesSearch; @@ -193,6 +195,12 @@ protected void init() { TypesSearch.and("types", TypesSearch.entity().getType(), Op.IN); TypesSearch.done(); + hypervisorTypeAndNonUserTypeSearch = createSearchBuilder(); + hypervisorTypeAndNonUserTypeSearch.and("hypervisorType", + hypervisorTypeAndNonUserTypeSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + hypervisorTypeAndNonUserTypeSearch.and("types", hypervisorTypeAndNonUserTypeSearch.entity().getType(), Op.NIN); + hypervisorTypeAndNonUserTypeSearch.done(); + IdTypesSearch = createSearchBuilder(); IdTypesSearch.and("id", IdTypesSearch.entity().getId(), Op.EQ); IdTypesSearch.and("types", IdTypesSearch.entity().getType(), Op.IN); @@ -427,6 +435,14 @@ public List listByTypes(Type... types) { return listBy(sc); } + @Override + public List listByHypervisorTypeAndNonUserTypes(HypervisorType hypervisorType) { + SearchCriteria sc = hypervisorTypeAndNonUserTypeSearch.create(); + sc.setParameters("hypervisorType", hypervisorType); + sc.setParameters("types", Type.User); + return listBy(sc); + } + @Override public List listByTypeAndState(VirtualMachine.Type type, State state) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 84c27583925b..6238484deef6 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -356,4 +356,5 @@ + diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 4cd2811cc204..a432744a8332 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -206,10 +206,12 @@ import org.apache.cloudstack.api.command.admin.systemvm.UpgradeSystemVMCmd; import org.apache.cloudstack.api.command.admin.template.CopyTemplateCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.CreateTemplateCmdByAdmin; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.admin.template.ListTemplatePermissionsCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.ListTemplatesCmdByAdmin; import org.apache.cloudstack.api.command.admin.template.PrepareTemplateCmd; import org.apache.cloudstack.api.command.admin.template.RegisterTemplateCmdByAdmin; +import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; import org.apache.cloudstack.api.command.admin.usage.AddTrafficMonitorCmd; import org.apache.cloudstack.api.command.admin.usage.AddTrafficTypeCmd; import org.apache.cloudstack.api.command.admin.usage.DeleteTrafficMonitorCmd; @@ -2842,6 +2844,8 @@ public List> getCommands() { cmdList.add(ListTemplatePermissionsCmd.class); cmdList.add(ListTemplatesCmd.class); cmdList.add(RegisterTemplateCmd.class); + cmdList.add(ActivateSystemVMTemplateCmd.class); + cmdList.add(GetSystemVMTemplateDefaultURLCmd.class); cmdList.add(UpdateTemplateCmd.class); cmdList.add(UpdateTemplatePermissionsCmd.class); cmdList.add(AddNicToVMCmd.class); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index df3b59b5ff73..69369f462285 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -20,10 +20,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -34,6 +36,7 @@ import com.cloud.deploy.DeployDestination; import com.cloud.storage.ImageStoreUploadMonitorImpl; +import com.cloud.upgrade.dao.VersionDao; import com.cloud.utils.StringUtils; import com.cloud.utils.EncryptionUtil; import com.cloud.utils.DateUtil; @@ -43,7 +46,10 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; @@ -281,6 +287,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, MessageBus _messageBus; @Inject private VMTemplateDetailsDao _tmpltDetailsDao; + @Inject + private VersionDao _versionDao; private boolean _disableExtraction = false; private List _adapters; @@ -292,6 +300,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject private EndPointSelector selector; + public static final String BASE_SYSTEMVM_URL = "https://download.cloudstack.org/systemvm"; + private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; @@ -403,6 +413,61 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT } } + @Override + public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd) { + VMTemplateVO template = _tmpltDao.findById(cmd.getId()); + if (template == null) { + throw new InvalidParameterValueException(String.format("Unable to find template with id %d" + cmd.getId())); + } + Transaction.execute(new TransactionCallbackNoReturn() { + + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + //Update the vm_template type field to ‘SYSTEM’ in database for the given id. + template.setTemplateType(TemplateType.SYSTEM); + boolean update = _tmpltDao.update(cmd.getId(), template); + //o Update the vm_instance records with the template ID for the specified hypervisor type in the database. + List vmInstanceVOS = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); + //o Update the router.template global configuration parameter in the database. + // + //o Update minreq.systemtemplate.version configuration parameter in the database. + } + + }); + + return template; + } + + @Override + public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd) { + String version = resolveVersion(cmd.getVersion()); + + List vmTemplateVOS = _tmpltDao.listSystemVMTemplatesByUrlLike(BASE_SYSTEMVM_URL + "%" + version, cmd.getHypervisor()); + if (!vmTemplateVOS.isEmpty()) { + Optional max = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); + return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), max.orElse(vmTemplateVOS.get(0)).getUrl()); + } else { + throw new CloudRuntimeException(String.format("cannot find System VM Template URL for version %s", version)); + } + } + + private String resolveVersion(String version) { + if (version == null || version.isEmpty()) { + version = _versionDao.getCurrentVersion(); + } + return formatVersion(version); + } + + private String formatVersion(String version) { + if (version == null) { + return ""; + } else if (version.split("[.]").length > 3) { + return version.substring(0, org.apache.commons.lang3.StringUtils.ordinalIndexOf(version, ".", 3)); + } else { + return version; + } + } + @Override public DataStore getImageStore(String storeUuid, Long zoneId) { From 169bcb77f155d36abf099f194c6f37d1b86bad80 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Fri, 12 Oct 2018 11:00:48 +0200 Subject: [PATCH 06/13] activateSystemVMTemplate api implementation logic --- .../cloud/template/TemplateManagerImpl.java | 69 ++++++++++++++++--- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 69369f462285..0ff7b90d64a8 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -50,7 +50,9 @@ import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; @@ -210,6 +212,13 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateHyperV; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateKvm; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateLxc; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateOvm3; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateVmware; +import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateXen; + public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable { private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class); @@ -417,7 +426,7 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd) { VMTemplateVO template = _tmpltDao.findById(cmd.getId()); if (template == null) { - throw new InvalidParameterValueException(String.format("Unable to find template with id %d" + cmd.getId())); + throw new InvalidParameterValueException(String.format("Unable to find template with id %d.", cmd.getId())); } Transaction.execute(new TransactionCallbackNoReturn() { @@ -425,19 +434,61 @@ public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateC public void doInTransactionWithoutResult(TransactionStatus status) { //Update the vm_template type field to ‘SYSTEM’ in database for the given id. template.setTemplateType(TemplateType.SYSTEM); - boolean update = _tmpltDao.update(cmd.getId(), template); - //o Update the vm_instance records with the template ID for the specified hypervisor type in the database. - List vmInstanceVOS = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); - //o Update the router.template global configuration parameter in the database. - // - //o Update minreq.systemtemplate.version configuration parameter in the database. - } + _tmpltDao.update(template.getId(), template); + + //Update the vm_instance records with the template ID for the specified hypervisor type in the database. + List vmInstances = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); + vmInstances.stream().forEach(instance -> { + instance.setTemplateId(template.getId()); + _vmInstanceDao.update(instance.getId(), instance); + }); + + //Update the router.template.x global configuration parameter in the database. + ConfigurationVO routerTemplate = _configDao.findByName(getRouterTemplateConfigKey(template.getHypervisorType())); + if (routerTemplate != null) { + routerTemplate.setValue(template.getName()); + _configDao.update(routerTemplate.getName(), routerTemplate); + } else { + throw new CloudRuntimeException(String.format("Cannot update router template configuration for hypervisor %s: unable to find it.", template.getHypervisorType())); + } + //Update minreq.systemtemplate.version configuration parameter in the database. + String version = formatVersion(_versionDao.getCurrentVersion()); + if (version.isEmpty()) { + throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find the current version.", NetworkOrchestrationService.MinVRVersion.key())); + } + ConfigurationVO minRequiredVersion = _configDao.findByName(NetworkOrchestrationService.MinVRVersion.key()); + if (minRequiredVersion != null) { + minRequiredVersion.setValue(version); + _configDao.update(minRequiredVersion.getName(), minRequiredVersion); + } else { + throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find it.", NetworkOrchestrationService.MinVRVersion.key())); + } + } }); return template; } + private String getRouterTemplateConfigKey(HypervisorType hypervisorType) { + switch (hypervisorType) { + case XenServer: + return RouterTemplateXen.key(); + case KVM: + return RouterTemplateKvm.key(); + case VMware: + return RouterTemplateVmware.key(); + case Hyperv: + return RouterTemplateHyperV.key(); + case LXC: + return RouterTemplateLxc.key(); + case Ovm3: + return RouterTemplateOvm3.key(); + default: + return ""; + } + } + @Override public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd) { String version = resolveVersion(cmd.getVersion()); @@ -447,7 +498,7 @@ public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSy Optional max = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), max.orElse(vmTemplateVOS.get(0)).getUrl()); } else { - throw new CloudRuntimeException(String.format("cannot find System VM Template URL for version %s", version)); + throw new CloudRuntimeException(String.format("Unable find System VM Template URL for version %s", version)); } } From 23193d8833461db87a226db9afac2468c1a5d43e Mon Sep 17 00:00:00 2001 From: ernjvr Date: Fri, 12 Oct 2018 11:41:58 +0200 Subject: [PATCH 07/13] clean up --- .../admin/template/ActivateSystemVMTemplateCmd.java | 5 ----- .../admin/template/GetSystemVMTemplateDefaultURLCmd.java | 7 ------- 2 files changed, 12 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java index 764879e30c5a..259351cf97c8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java @@ -59,7 +59,6 @@ public Long getId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { -// try { VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(this); if (template != null) { ListResponse response = new ListResponse(); @@ -71,10 +70,6 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to activate template."); } -// } catch ( ex) { -// LOGGER.warn("Exception: ", ex); -// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); -// } } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java index 7638dd0e94b2..2057f5278332 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/GetSystemVMTemplateDefaultURLCmd.java @@ -62,13 +62,6 @@ public String getHypervisor() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { -// VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, 1); -// String url = template.getUrl(); -// if (url != null && !url.isEmpty()) { -// setResponseObject(new GetSystemVMTemplateDefaultURLResponse(getCommandName(), url)); -// } else { -// throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to find URL for version '%s' and hypervisor '%s'", version, hypervisor)); -// } GetSystemVMTemplateDefaultURLResponse response = _templateService.getSystemVMTemplateDefaultURL(this); if (response != null) { setResponseObject(response); From 79f2eb8af073d612bd4f3156ead14356ff6deb0f Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 17 Oct 2018 17:28:45 +0200 Subject: [PATCH 08/13] enhance register system vm api & refactor other --- .../cloud/template/TemplateApiService.java | 3 +- .../apache/cloudstack/api/ApiConstants.java | 2 + .../template/ActivateSystemVMTemplateCmd.java | 2 +- .../user/template/RegisterTemplateCmd.java | 25 +++- .../storage/template/TemplateConstants.java | 2 +- .../template/HypervisorTemplateAdapter.java | 1 + .../cloud/template/TemplateAdapterBase.java | 25 +++- .../cloud/template/TemplateManagerImpl.java | 118 ++++++++++-------- 8 files changed, 112 insertions(+), 66 deletions(-) diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index b9c187a8d299..16d61bf89a13 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -22,7 +22,6 @@ import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; -import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; @@ -52,7 +51,7 @@ public interface TemplateApiService { GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd); - VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd); + VirtualMachineTemplate activateSystemVMTemplate(long templateId); VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index b7779cb2a8d9..34d30cd803b3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -299,6 +299,7 @@ public class ApiConstants { public static final String TARGET_IQN = "targetiqn"; public static final String TEMPLATE_FILTER = "templatefilter"; public static final String TEMPLATE_ID = "templateid"; + public static final String TEMPLATE_TYPE = "templatetype"; public static final String ISO_ID = "isoid"; public static final String TIMEOUT = "timeout"; public static final String TIMEZONE = "timezone"; @@ -602,6 +603,7 @@ public class ApiConstants { public static final String INTERVAL = "interval"; public static final String QUIETTIME = "quiettime"; public static final String ACTION = "action"; + public static final String ACTIVATE = "activate"; public static final String CONDITION_ID = "conditionid"; public static final String CONDITION_IDS = "conditionids"; public static final String COUNTERPARAM_LIST = "counterparam"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java index 259351cf97c8..36097d67b9f2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ActivateSystemVMTemplateCmd.java @@ -59,7 +59,7 @@ public Long getId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(this); + VirtualMachineTemplate template = _templateService.activateSystemVMTemplate(id); if (template != null) { ListResponse response = new ListResponse(); List templateResponses = _responseGenerator.createTemplateResponses(ResponseObject.ResponseView.Restricted, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 333b363d16a9..6cd6133a11d3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -22,7 +22,9 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; +import com.cloud.storage.Storage.TemplateType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; import org.apache.cloudstack.api.ApiConstants; @@ -161,6 +163,13 @@ public class RegisterTemplateCmd extends BaseCmd { description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment") private Boolean directDownload; + @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING, description = "the template type. Possible values include user and system.") + private String templateType; + + @Parameter(name=ApiConstants.ACTIVATE, type = CommandType.BOOLEAN, + description = "true if this template should be used by CloudStack to create System VMs. Must be used with templatetype of 'system'.") + private Boolean activate; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -273,6 +282,14 @@ public boolean isDirectDownload() { return directDownload == null ? false : directDownload; } + public String getTemplateType() { + return Optional.ofNullable(templateType).orElse(""); + } + + public Boolean isActivate() { + return Optional.ofNullable(activate).orElse(false); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -319,11 +336,17 @@ public void execute() throws ResourceAllocationException { } protected void validateParameters() { + if ((!getTemplateType().isEmpty() && !getTemplateType().equalsIgnoreCase(TemplateType.USER.name()) + && !getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name()))) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Invalid parameter value specified for templateType. Valid values: user or system"); + } if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty())) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Both zoneid and zoneids cannot be specified at the same time"); - if (zoneId == null && (zoneIds == null || zoneIds.isEmpty())) + if ((zoneId == null && (zoneIds == null || zoneIds.isEmpty())) && + (!getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name()))) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Either zoneid or zoneids is required. Both cannot be null."); diff --git a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java index 25c2d5b3c074..55be0b2c409d 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java @@ -26,7 +26,7 @@ public final class TemplateConstants { public static final String DEFAULT_TMPLT_FIRST_LEVEL_DIR = "tmpl/"; public static final String DEFAULT_SYSTEM_VM_TEMPLATE_PATH = "template/tmpl/1/"; - + public static final String DEFAULT_BASE_SYSTEMVM_URL = "https://download.cloudstack.org/systemvm"; public static final String DEFAULT_SYSTEM_VM_TMPLT_NAME = "routing"; public static final int DEFAULT_TMPLT_COPY_PORT = 80; diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 8aa21661675f..ac78e84116c5 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -171,6 +171,7 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio TemplateProfile profile = super.prepare(cmd); String url = profile.getUrl(); UriUtils.validateUrl(cmd.getFormat(), url); + UriUtils.checkUrlExistence(url); if (cmd.isDirectDownload()) { DigestHelper.validateChecksumString(cmd.getChecksum()); Long templateSize = performDirectDownloadUrlValidation(url); diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index ebb73daa590d..a5830552a59b 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -159,7 +159,9 @@ public TemplateProfile prepare(boolean isIso, long userId, String name, String d if (passwordEnabled == null) { passwordEnabled = false; } - if (requiresHVM == null) { + if (templateType == TemplateType.SYSTEM) { + requiresHVM = false; + } else if (requiresHVM == null) { requiresHVM = true; } } @@ -262,13 +264,12 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio Account caller = CallContext.current().getCallingAccount(); Account owner = _accountMgr.getAccount(cmd.getEntityOwnerId()); _accountMgr.checkAccess(caller, null, true, owner); - - boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); + TemplateType templateType = getTemplateType(cmd); List zoneId = cmd.getZoneIds(); - // ignore passed zoneId if we are using region wide image store + // ignore passed zoneId if we are using region wide image store or system template type List stores = _imgStoreDao.findRegionImageStores(); - if (stores != null && stores.size() > 0) { + if ((stores != null && stores.size() > 0) || templateType.equals(TemplateType.SYSTEM)){ zoneId = null; } @@ -280,8 +281,20 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(), cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true, - cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload()); + cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), templateType, cmd.isDirectDownload()); + + } + + private TemplateType getTemplateType(RegisterTemplateCmd cmd) { + boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); + if (cmd.getTemplateType() != null && cmd.getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name())) { + return TemplateType.SYSTEM; + } else if (isRouting) { + return TemplateType.ROUTING; + } else { + return TemplateType.USER; + } } @Override diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 0ff7b90d64a8..371941489959 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -46,7 +46,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.cloudstack.api.command.admin.template.ActivateSystemVMTemplateCmd; import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; @@ -218,6 +217,7 @@ import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateOvm3; import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateVmware; import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateXen; +import static com.cloud.storage.template.TemplateConstants.DEFAULT_BASE_SYSTEMVM_URL; public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable { private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class); @@ -309,8 +309,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject private EndPointSelector selector; - public static final String BASE_SYSTEMVM_URL = "https://download.cloudstack.org/systemvm"; - private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; @@ -356,16 +354,22 @@ public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws U throw new PermissionDeniedException("Parameter isrouting can only be specified by a Root Admin, permission denied"); } } + if (cmd.getTemplateType().equalsIgnoreCase(Storage.TemplateType.SYSTEM.name())) { + if (!_accountService.isRootAdmin(account.getId())) { + throw new PermissionDeniedException(String.format("Value of '%s' for parameter templatetype can only be specified by a Root Admin, permission denied", + Storage.TemplateType.SYSTEM.name())); + } + } TemplateAdapter adapter = getAdapter(HypervisorType.getType(cmd.getHypervisor())); TemplateProfile profile = adapter.prepare(cmd); - VMTemplateVO template = adapter.create(profile); + VMTemplateVO template = Optional.ofNullable(adapter.create(profile)).orElseThrow(() -> new CloudRuntimeException("Failed to create a template")); - if (template != null) { - return template; - } else { - throw new CloudRuntimeException("Failed to create a template"); + if (cmd.getTemplateType().equalsIgnoreCase(Storage.TemplateType.SYSTEM.name()) + && cmd.isActivate()) { + return activateSystemVMTemplate(template.getId()); } + return template; } @Override @@ -423,53 +427,55 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT } @Override - public VirtualMachineTemplate activateSystemVMTemplate(ActivateSystemVMTemplateCmd cmd) { - VMTemplateVO template = _tmpltDao.findById(cmd.getId()); - if (template == null) { - throw new InvalidParameterValueException(String.format("Unable to find template with id %d.", cmd.getId())); - } + public VirtualMachineTemplate activateSystemVMTemplate(long templateId) { + VMTemplateVO template = Optional.ofNullable(_tmpltDao.findById(templateId)) + .orElseThrow(() -> new InvalidParameterValueException(String.format("Unable to find template with id %d.", templateId))); Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - //Update the vm_template type field to ‘SYSTEM’ in database for the given id. - template.setTemplateType(TemplateType.SYSTEM); - _tmpltDao.update(template.getId(), template); - - //Update the vm_instance records with the template ID for the specified hypervisor type in the database. - List vmInstances = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); - vmInstances.stream().forEach(instance -> { - instance.setTemplateId(template.getId()); - _vmInstanceDao.update(instance.getId(), instance); - }); - - //Update the router.template.x global configuration parameter in the database. - ConfigurationVO routerTemplate = _configDao.findByName(getRouterTemplateConfigKey(template.getHypervisorType())); - if (routerTemplate != null) { - routerTemplate.setValue(template.getName()); - _configDao.update(routerTemplate.getName(), routerTemplate); - } else { - throw new CloudRuntimeException(String.format("Cannot update router template configuration for hypervisor %s: unable to find it.", template.getHypervisorType())); - } - - //Update minreq.systemtemplate.version configuration parameter in the database. - String version = formatVersion(_versionDao.getCurrentVersion()); - if (version.isEmpty()) { - throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find the current version.", NetworkOrchestrationService.MinVRVersion.key())); - } - ConfigurationVO minRequiredVersion = _configDao.findByName(NetworkOrchestrationService.MinVRVersion.key()); - if (minRequiredVersion != null) { - minRequiredVersion.setValue(version); - _configDao.update(minRequiredVersion.getName(), minRequiredVersion); - } else { - throw new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find it.", NetworkOrchestrationService.MinVRVersion.key())); - } + updateTemplate(template); + updateVMInstances(template); + updateRouterTemplateConfig(template); + updateMinRequiredSystemVMVersionConfig(); } }); return template; } + private void updateTemplate(VMTemplateVO template) { + template.setTemplateType(TemplateType.SYSTEM); + _tmpltDao.update(template.getId(), template); + } + + private void updateVMInstances(VMTemplateVO template) { + List vmInstances = _vmInstanceDao.listByHypervisorTypeAndNonUserTypes(template.getHypervisorType()); + vmInstances.stream().forEach(instance -> updateVmInstance(instance, template)); + } + + private void updateVmInstance(VMInstanceVO instance, VMTemplateVO template) { + instance.setTemplateId(template.getId()); + _vmInstanceDao.update(instance.getId(), instance); + } + + private void updateRouterTemplateConfig(VMTemplateVO template) { + String routerTemplateConfigKey = getRouterTemplateConfigKey(template.getHypervisorType()); + ConfigurationVO routerTemplate = Optional.ofNullable(_configDao.findByName(routerTemplateConfigKey)) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration for hypervisor %s: unable to find it.", routerTemplateConfigKey, template.getHypervisorType()))); + routerTemplate.setValue(template.getName()); + _configDao.update(routerTemplate.getName(), routerTemplate); + } + + private void updateMinRequiredSystemVMVersionConfig() { + String version = formatVersion(Optional.ofNullable(_versionDao.getCurrentVersion()) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find the current version.", NetworkOrchestrationService.MinVRVersion.key())))); + ConfigurationVO minRequiredVersion = Optional.ofNullable(_configDao.findByName(NetworkOrchestrationService.MinVRVersion.key())) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot update %s configuration: unable to find it.", NetworkOrchestrationService.MinVRVersion.key()))); + minRequiredVersion.setValue(version); + _configDao.update(minRequiredVersion.getName(), minRequiredVersion); + } + private String getRouterTemplateConfigKey(HypervisorType hypervisorType) { switch (hypervisorType) { case XenServer: @@ -491,28 +497,30 @@ private String getRouterTemplateConfigKey(HypervisorType hypervisorType) { @Override public GetSystemVMTemplateDefaultURLResponse getSystemVMTemplateDefaultURL(GetSystemVMTemplateDefaultURLCmd cmd) { - String version = resolveVersion(cmd.getVersion()); + List vmTemplateVOS = getSystemVMTemplatesByUrlVersionMatch(cmd); + Optional maxIdTemplate = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); + return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), maxIdTemplate.orElse(vmTemplateVOS.get(0)).getUrl()); + } - List vmTemplateVOS = _tmpltDao.listSystemVMTemplatesByUrlLike(BASE_SYSTEMVM_URL + "%" + version, cmd.getHypervisor()); + private List getSystemVMTemplatesByUrlVersionMatch(GetSystemVMTemplateDefaultURLCmd cmd) { + String version = resolveVersion(Optional.ofNullable(cmd.getVersion()).orElse("")); + List vmTemplateVOS = _tmpltDao.listSystemVMTemplatesByUrlLike(DEFAULT_BASE_SYSTEMVM_URL + "%" + version, cmd.getHypervisor()); if (!vmTemplateVOS.isEmpty()) { - Optional max = vmTemplateVOS.stream().max(Comparator.comparing(VMTemplateVO::getId)); - return new GetSystemVMTemplateDefaultURLResponse(cmd.getCommandName(), max.orElse(vmTemplateVOS.get(0)).getUrl()); + return vmTemplateVOS; } else { - throw new CloudRuntimeException(String.format("Unable find System VM Template URL for version %s", version)); + throw new CloudRuntimeException(String.format("Unable find System VM Template URL for version %s and hypervisor %s.", version, cmd.getHypervisor())); } } private String resolveVersion(String version) { - if (version == null || version.isEmpty()) { - version = _versionDao.getCurrentVersion(); + if (version.isEmpty()) { + version = Optional.ofNullable(_versionDao.getCurrentVersion()).orElse(""); } return formatVersion(version); } private String formatVersion(String version) { - if (version == null) { - return ""; - } else if (version.split("[.]").length > 3) { + if (version.split("[.]").length > 3) { return version.substring(0, org.apache.commons.lang3.StringUtils.ordinalIndexOf(version, ".", 3)); } else { return version; From 75432aca270f8f3acfe7de0f2dcb43132b878b5d Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 24 Oct 2018 14:09:07 +0200 Subject: [PATCH 09/13] add mount logic untested --- .../apache/cloudstack/api/ApiConstants.java | 2 +- .../user/template/RegisterTemplateCmd.java | 17 +- .../storage/download/DownloadMonitorImpl.java | 62 ++++++- .../download/ManagementServerDownloader.java | 166 ++++++++++++++++++ .../cloud/template/TemplateAdapterBase.java | 2 +- .../cloud/template/TemplateManagerImpl.java | 9 +- .../spring-server-core-managers-context.xml | 4 +- 7 files changed, 239 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 34d30cd803b3..3715e97999e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -299,7 +299,7 @@ public class ApiConstants { public static final String TARGET_IQN = "targetiqn"; public static final String TEMPLATE_FILTER = "templatefilter"; public static final String TEMPLATE_ID = "templateid"; - public static final String TEMPLATE_TYPE = "templatetype"; + public static final String SYSTEM = "system"; public static final String ISO_ID = "isoid"; public static final String TIMEOUT = "timeout"; public static final String TIMEZONE = "timezone"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 6cd6133a11d3..81a731e2ad89 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Optional; -import com.cloud.storage.Storage.TemplateType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; import org.apache.cloudstack.api.ApiConstants; @@ -163,8 +162,8 @@ public class RegisterTemplateCmd extends BaseCmd { description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment") private Boolean directDownload; - @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING, description = "the template type. Possible values include user and system.") - private String templateType; + @Parameter(name = ApiConstants.SYSTEM, type = CommandType.BOOLEAN, description = "true if it is a system vm template.") + private Boolean system; @Parameter(name=ApiConstants.ACTIVATE, type = CommandType.BOOLEAN, description = "true if this template should be used by CloudStack to create System VMs. Must be used with templatetype of 'system'.") @@ -282,8 +281,8 @@ public boolean isDirectDownload() { return directDownload == null ? false : directDownload; } - public String getTemplateType() { - return Optional.ofNullable(templateType).orElse(""); + public Boolean isSystem() { + return Optional.ofNullable(system).orElse(false); } public Boolean isActivate() { @@ -336,17 +335,11 @@ public void execute() throws ResourceAllocationException { } protected void validateParameters() { - if ((!getTemplateType().isEmpty() && !getTemplateType().equalsIgnoreCase(TemplateType.USER.name()) - && !getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name()))) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, - "Invalid parameter value specified for templateType. Valid values: user or system"); - } if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty())) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Both zoneid and zoneids cannot be specified at the same time"); - if ((zoneId == null && (zoneIds == null || zoneIds.isEmpty())) && - (!getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name()))) + if ((zoneId == null && (zoneIds == null || zoneIds.isEmpty())) && !isSystem()) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Either zoneid or zoneids is required. Both cannot be null."); diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 68be52d842fb..7fff5f95da92 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -19,12 +19,15 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Timer; import javax.inject.Inject; +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.Storage; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -81,11 +84,16 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor private ConfigurationDao _configDao; @Inject private EndPointSelector _epSelector; + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; + @Inject + private ManagementServerDownloader managementServerDownloader; private String _copyAuthPasswd; private String _proxy = null; private Timer _timer; + private final Map _storageMounts = new HashMap<>(); @Override public boolean configure(String name, Map params) { @@ -151,10 +159,15 @@ private void initiateTemplateDownload(DataObject template, AsyncCompletionCallba dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd); } EndPoint ep = _epSelector.select(template); + downloadUsingManagementServer(template, callback); if (ep == null) { - String errMsg = "There is no secondary storage VM for downloading template to image store " + store.getName(); - s_logger.warn(errMsg); - throw new CloudRuntimeException(errMsg); + if (tmpl.getTemplateType() == Storage.TemplateType.SYSTEM) { + downloadUsingManagementServer(template, callback); + } else { + String errMsg = "There is no secondary storage VM for downloading template to image store " + store.getName(); + s_logger.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } } DownloadListener dl = new DownloadListener(ep, store, template, _timer, this, dcmd, callback); ComponentContext.inject(dl); // initialize those auto-wired field in download listener. @@ -178,6 +191,49 @@ private void initiateTemplateDownload(DataObject template, AsyncCompletionCallba } } + private void downloadUsingManagementServer(DataObject template, AsyncCompletionCallback callback) { + Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(template.getDataStore().getId()); + String mountPoint = managementServerDownloader.getMountPoint(template.getDataStore().getUri(), nfsVersion); + +// Answer answer = new DownloadAnswer(null, true, "download complete") +// callback.complete(answer); + } + +// private File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String name, DataObject template) { +// HttpClient client = new DefaultHttpClient(); +// HttpGet get = new HttpGet(url); +// try { +// HttpResponse response = client.execute(get); +// HttpEntity entity = response.getEntity(); +// if (entity == null) { +// s_logger.debug("Faled to get entity"); +// throw new CloudRuntimeException("Failed to get url: " + url); +// } +// +// String nfsMountPath = getRootDir(nfs.getUrl(), _nfsVersion); +// +// String filePath = nfsMountPath + File.separator + path; +// File directory = new File(filePath); +// if (!directory.exists()) { +// template.getDataStore().mkdirs(filePath); +// } +// File destFile = new File(filePath + File.separator + name); +// if (!destFile.createNewFile()) { +// s_logger.warn("Reusing existing file " + destFile.getPath()); +// } +// try (FileOutputStream outputStream = new FileOutputStream(destFile);) { +// entity.writeTo(outputStream); +// } catch (IOException e) { +// s_logger.debug("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); +// } +// return new File(destFile.getAbsolutePath()); +// } catch (IOException e) { +// s_logger.debug("Faild to get url:" + url + ", due to " + e.toString()); +// throw new CloudRuntimeException(e); +// } +// } + + @Override public void downloadTemplateToStorage(DataObject template, AsyncCompletionCallback callback) { if(template != null) { diff --git a/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java b/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java new file mode 100644 index 000000000000..af4517ae7b12 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java @@ -0,0 +1,166 @@ +package com.cloud.storage.download; + +import com.cloud.storage.JavaStorageLayer; +import com.cloud.storage.StorageLayer; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class ManagementServerDownloader { + private static final Logger s_logger = Logger.getLogger(ManagementServerDownloader.class); + private StorageLayer _storage; + private int _timeout; + private final Random _rand = new Random(System.currentTimeMillis()); + private final Map _storageMounts = new HashMap<>(); + + public static final ConfigKey MOUNT_PARENT = new ConfigKey<>("Advanced", String.class, + "mount.parent", "/var/cloudstack/mnt", + "The mount point on the Management Server for Secondary Storage.", + true, ConfigKey.Scope.Global); + + public ManagementServerDownloader() { + this._storage = new JavaStorageLayer(); + _timeout = 1440 * 1000; + } + + public String getMountPoint(String storageUrl, Integer nfsVersion) { + String mountPoint = null; + synchronized (_storageMounts) { + mountPoint = _storageMounts.get(storageUrl); + if (mountPoint != null) { + return mountPoint; + } + + URI uri; + try { + uri = new URI(storageUrl); + } catch (URISyntaxException e) { + s_logger.error("Invalid storage URL format ", e); + throw new CloudRuntimeException("Unable to create mount point due to invalid storage URL format " + storageUrl); + } + + mountPoint = mount(uri.getHost() + ":" + uri.getPath(), MOUNT_PARENT.value(), nfsVersion); + if (mountPoint == null) { + s_logger.error("Unable to create mount point for " + storageUrl); + return "/mnt/sec"; // throw new CloudRuntimeException("Unable to create mount point for " + storageUrl); + } + + _storageMounts.put(storageUrl, mountPoint); + return mountPoint; + } + } + + private String mount(String path, String parent, Integer nfsVersion) { + String mountPoint = setupMountPoint(parent); + if (mountPoint == null) { + s_logger.warn("Unable to create a mount point"); + return null; + } + + Script script = null; + String result = null; + Script command = new Script(true, "mount", _timeout, s_logger); + command.add("-t", "nfs"); + if (nfsVersion != null){ + command.add("-o", "vers=" + nfsVersion); + } + // command.add("-o", "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0"); + if ("Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) { + command.add("-o", "resvport"); + } + command.add(path); + command.add(mountPoint); + result = command.execute(); + if (result != null) { + s_logger.warn("Unable to mount " + path + " due to " + result); + File file = new File(mountPoint); + if (file.exists()) { + file.delete(); + } + return null; + } + + // Change permissions for the mountpoint + script = new Script(true, "chmod", _timeout, s_logger); + script.add("1777", mountPoint); + result = script.execute(); + if (result != null) { + s_logger.warn("Unable to set permissions for " + mountPoint + " due to " + result); + } + return mountPoint; + } + + private String setupMountPoint(String parent) { + String mountPoint = null; + long mshostId = ManagementServerNode.getManagementServerId(); + for (int i = 0; i < 10; i++) { + String mntPt = parent + File.separator + String.valueOf(mshostId) + "." + Integer.toHexString(_rand.nextInt(Integer.MAX_VALUE)); + File file = new File(mntPt); + if (!file.exists()) { + if (_storage.mkdir(mntPt)) { + mountPoint = mntPt; + break; + } + } + s_logger.error("Unable to create mount: " + mntPt); + } + + return mountPoint; + } + protected boolean mountExists(String localRootPath, URI uri) { + Script script = null; + script = new Script(true, "mount", _timeout, s_logger); + + List res = new ArrayList(); + ZfsPathParser parser = new ZfsPathParser(localRootPath); + script.execute(parser); + res.addAll(parser.getPaths()); + for (String s : res) { + if (s.contains(localRootPath)) { + s_logger.debug("Some device already mounted at " + localRootPath + ", no need to mount " + uri.toString()); + return true; + } + } + return false; + } + public static class ZfsPathParser extends OutputInterpreter { + String _parent; + List paths = new ArrayList(); + + public ZfsPathParser(String parent) { + _parent = parent; + } + + @Override + public String interpret(BufferedReader reader) throws IOException { + String line = null; + while ((line = reader.readLine()) != null) { + paths.add(line); + } + return null; + } + + public List getPaths() { + return paths; + } + + @Override + public boolean drain() { + return true; + } + } +} diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index a5830552a59b..3feb01a391c1 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -288,7 +288,7 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio private TemplateType getTemplateType(RegisterTemplateCmd cmd) { boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); - if (cmd.getTemplateType() != null && cmd.getTemplateType().equalsIgnoreCase(TemplateType.SYSTEM.name())) { + if (cmd.isSystem()) { return TemplateType.SYSTEM; } else if (isRouting) { return TemplateType.ROUTING; diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 371941489959..8858367b6232 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -354,10 +354,10 @@ public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws U throw new PermissionDeniedException("Parameter isrouting can only be specified by a Root Admin, permission denied"); } } - if (cmd.getTemplateType().equalsIgnoreCase(Storage.TemplateType.SYSTEM.name())) { + if (cmd.isSystem()) { if (!_accountService.isRootAdmin(account.getId())) { - throw new PermissionDeniedException(String.format("Value of '%s' for parameter templatetype can only be specified by a Root Admin, permission denied", - Storage.TemplateType.SYSTEM.name())); + throw new PermissionDeniedException(String.format("Value of '%s' for parameter %s can only be specified by a Root Admin, permission denied", + cmd.isSystem(), ApiConstants.SYSTEM)); } } @@ -365,8 +365,7 @@ public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws U TemplateProfile profile = adapter.prepare(cmd); VMTemplateVO template = Optional.ofNullable(adapter.create(profile)).orElseThrow(() -> new CloudRuntimeException("Failed to create a template")); - if (cmd.getTemplateType().equalsIgnoreCase(Storage.TemplateType.SYSTEM.name()) - && cmd.isActivate()) { + if (cmd.isSystem() && cmd.isActivate()) { return activateSystemVMTemplate(template.getId()); } return template; diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 2f67c4248d35..965a9b8ab426 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -248,7 +248,9 @@ - + + + From 1ddf2895804a29cd8a41f1af015dc3b2ae2d23c0 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Fri, 2 Nov 2018 10:19:56 +0200 Subject: [PATCH 10/13] wip download logic and nfs refactor --- .../storage/download/DownloadMonitorImpl.java | 140 ++++------- .../download/ManagementServerDownloader.java | 226 +++++++---------- .../HttpTemplateDownloader.java | 75 ++++++ .../managementserver/TemplateDownloader.java | 77 ++++++ .../TemplateDownloaderImpl.java | 231 ++++++++++++++++++ .../com/cloud/storage/mount/MountManager.java | 5 + .../cloud/storage/mount/NfsMountManager.java | 194 +++++++++++++++ .../spring-server-core-managers-context.xml | 6 + 8 files changed, 724 insertions(+), 230 deletions(-) create mode 100644 server/src/main/java/com/cloud/storage/download/managementserver/HttpTemplateDownloader.java create mode 100644 server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloader.java create mode 100644 server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloaderImpl.java create mode 100644 server/src/main/java/com/cloud/storage/mount/MountManager.java create mode 100644 server/src/main/java/com/cloud/storage/mount/NfsMountManager.java diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 7fff5f95da92..d26c68b486c3 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -16,21 +16,24 @@ // under the License. package com.cloud.storage.download; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Timer; - -import javax.inject.Inject; - +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.configuration.Config; import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.Storage; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.Volume; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.storage.upload.UploadListener; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -48,23 +51,17 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; -import com.cloud.agent.AgentManager; -import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; -import com.cloud.configuration.Config; -import com.cloud.storage.RegisterVolumePayload; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import com.cloud.storage.Volume; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.storage.template.TemplateConstants; -import com.cloud.storage.upload.UploadListener; -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.utils.component.ComponentContext; -import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Timer; @Component public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor { @@ -159,81 +156,40 @@ private void initiateTemplateDownload(DataObject template, AsyncCompletionCallba dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd); } EndPoint ep = _epSelector.select(template); - downloadUsingManagementServer(template, callback); + ep = null;//just for test -> remove if (ep == null) { if (tmpl.getTemplateType() == Storage.TemplateType.SYSTEM) { - downloadUsingManagementServer(template, callback); + managementServerDownloader.download(tmpl, template); + //set to ready state } else { String errMsg = "There is no secondary storage VM for downloading template to image store " + store.getName(); s_logger.warn(errMsg); throw new CloudRuntimeException(errMsg); } - } - DownloadListener dl = new DownloadListener(ep, store, template, _timer, this, dcmd, callback); - ComponentContext.inject(dl); // initialize those auto-wired field in download listener. - if (downloadJobExists) { - // due to handling existing download job issues, we still keep - // downloadState in template_store_ref to avoid big change in - // DownloadListener to use - // new ObjectInDataStore.State transition. TODO: fix this later - // to be able to remove downloadState from template_store_ref. - s_logger.info("found existing download job"); - dl.setCurrState(vmTemplateStore.getDownloadState()); - } + } else { + DownloadListener dl = new DownloadListener(ep, store, template, _timer, this, dcmd, callback); + ComponentContext.inject(dl); // initialize those auto-wired field in download listener. + if (downloadJobExists) { + // due to handling existing download job issues, we still keep + // downloadState in template_store_ref to avoid big change in + // DownloadListener to use + // new ObjectInDataStore.State transition. TODO: fix this later + // to be able to remove downloadState from template_store_ref. + s_logger.info("found existing download job"); + dl.setCurrState(vmTemplateStore.getDownloadState()); + } - try { - ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); - } catch (Exception e) { - s_logger.warn("Unable to start /resume download of template " + template.getId() + " to " + store.getName(), e); - dl.setDisconnected(); - dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + try { + ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); + } catch (Exception e) { + s_logger.warn("Unable to start /resume download of template " + template.getId() + " to " + store.getName(), e); + dl.setDisconnected(); + dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + } } } } - private void downloadUsingManagementServer(DataObject template, AsyncCompletionCallback callback) { - Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(template.getDataStore().getId()); - String mountPoint = managementServerDownloader.getMountPoint(template.getDataStore().getUri(), nfsVersion); - -// Answer answer = new DownloadAnswer(null, true, "download complete") -// callback.complete(answer); - } - -// private File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String name, DataObject template) { -// HttpClient client = new DefaultHttpClient(); -// HttpGet get = new HttpGet(url); -// try { -// HttpResponse response = client.execute(get); -// HttpEntity entity = response.getEntity(); -// if (entity == null) { -// s_logger.debug("Faled to get entity"); -// throw new CloudRuntimeException("Failed to get url: " + url); -// } -// -// String nfsMountPath = getRootDir(nfs.getUrl(), _nfsVersion); -// -// String filePath = nfsMountPath + File.separator + path; -// File directory = new File(filePath); -// if (!directory.exists()) { -// template.getDataStore().mkdirs(filePath); -// } -// File destFile = new File(filePath + File.separator + name); -// if (!destFile.createNewFile()) { -// s_logger.warn("Reusing existing file " + destFile.getPath()); -// } -// try (FileOutputStream outputStream = new FileOutputStream(destFile);) { -// entity.writeTo(outputStream); -// } catch (IOException e) { -// s_logger.debug("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); -// } -// return new File(destFile.getAbsolutePath()); -// } catch (IOException e) { -// s_logger.debug("Faild to get url:" + url + ", due to " + e.toString()); -// throw new CloudRuntimeException(e); -// } -// } - - @Override public void downloadTemplateToStorage(DataObject template, AsyncCompletionCallback callback) { if(template != null) { diff --git a/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java b/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java index af4517ae7b12..b4576d02aeb4 100644 --- a/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java +++ b/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java @@ -1,166 +1,116 @@ package com.cloud.storage.download; -import com.cloud.storage.JavaStorageLayer; -import com.cloud.storage.StorageLayer; +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.download.managementserver.HttpTemplateDownloader; +import com.cloud.storage.download.managementserver.TemplateDownloader; +import com.cloud.storage.mount.MountManager; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.OutputInterpreter; -import com.cloud.utils.script.Script; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.log4j.Logger; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; +import javax.inject.Inject; +import java.util.Date; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; public class ManagementServerDownloader { private static final Logger s_logger = Logger.getLogger(ManagementServerDownloader.class); - private StorageLayer _storage; - private int _timeout; - private final Random _rand = new Random(System.currentTimeMillis()); - private final Map _storageMounts = new HashMap<>(); - public static final ConfigKey MOUNT_PARENT = new ConfigKey<>("Advanced", String.class, - "mount.parent", "/var/cloudstack/mnt", - "The mount point on the Management Server for Secondary Storage.", - true, ConfigKey.Scope.Global); + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; - public ManagementServerDownloader() { - this._storage = new JavaStorageLayer(); - _timeout = 1440 * 1000; - } + @Inject + VMTemplatePoolDao vmTemplatePoolDao; - public String getMountPoint(String storageUrl, Integer nfsVersion) { - String mountPoint = null; - synchronized (_storageMounts) { - mountPoint = _storageMounts.get(storageUrl); - if (mountPoint != null) { - return mountPoint; - } + @Inject + private TemplateDataStoreDao _vmTemplateStoreDao; - URI uri; - try { - uri = new URI(storageUrl); - } catch (URISyntaxException e) { - s_logger.error("Invalid storage URL format ", e); - throw new CloudRuntimeException("Unable to create mount point due to invalid storage URL format " + storageUrl); - } + @Inject + private MountManager mountManager; - mountPoint = mount(uri.getHost() + ":" + uri.getPath(), MOUNT_PARENT.value(), nfsVersion); - if (mountPoint == null) { - s_logger.error("Unable to create mount point for " + storageUrl); - return "/mnt/sec"; // throw new CloudRuntimeException("Unable to create mount point for " + storageUrl); - } + private static final ExecutorService THREAD_SERVICE = Executors.newCachedThreadPool(new NamedThreadFactory("ManagementServerDownloader")); - _storageMounts.put(storageUrl, mountPoint); - return mountPoint; - } + public void download(VirtualMachineTemplate tmpl, DataObject template) { + downloadFromUrlToNfs(tmpl, template, template.getUuid()); } - private String mount(String path, String parent, Integer nfsVersion) { - String mountPoint = setupMountPoint(parent); - if (mountPoint == null) { - s_logger.warn("Unable to create a mount point"); - return null; - } - - Script script = null; - String result = null; - Script command = new Script(true, "mount", _timeout, s_logger); - command.add("-t", "nfs"); - if (nfsVersion != null){ - command.add("-o", "vers=" + nfsVersion); - } - // command.add("-o", "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0"); - if ("Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) { - command.add("-o", "resvport"); - } - command.add(path); - command.add(mountPoint); - result = command.execute(); - if (result != null) { - s_logger.warn("Unable to mount " + path + " due to " + result); - File file = new File(mountPoint); - if (file.exists()) { - file.delete(); - } - return null; - } - // Change permissions for the mountpoint - script = new Script(true, "chmod", _timeout, s_logger); - script.add("1777", mountPoint); - result = script.execute(); - if (result != null) { - s_logger.warn("Unable to set permissions for " + mountPoint + " due to " + result); - } - return mountPoint; - } - - private String setupMountPoint(String parent) { - String mountPoint = null; - long mshostId = ManagementServerNode.getManagementServerId(); - for (int i = 0; i < 10; i++) { - String mntPt = parent + File.separator + String.valueOf(mshostId) + "." + Integer.toHexString(_rand.nextInt(Integer.MAX_VALUE)); - File file = new File(mntPt); - if (!file.exists()) { - if (_storage.mkdir(mntPt)) { - mountPoint = mntPt; - break; + public void downloadFromUrlToNfs(VirtualMachineTemplate tmpl, DataObject template, String fileName) { + Runnable downloadFile = () -> { + Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(template.getDataStore().getId()); + String mountPoint = mountManager.getMountPoint(template.getDataStore().getUri(), nfsVersion); + HttpTemplateDownloader downloader = new HttpTemplateDownloader(tmpl, mountPoint); + if (downloader.downloadTemplate()) { + if (downloader.extractAndInstallDownloadedTemplate()) { + TemplateDownloader.TemplateInformation info = downloader.getTemplateInformation(); + try(TransactionLegacy txn = TransactionLegacy.open(ManagementServerDownloader.class.getName())) { + persistTemplateStorePoolRef(tmpl, template.getDataStore().getId(), info); + persistTemplateStoreRef(template, info); + } + } else { + throw new CloudRuntimeException(String.format("Failed to extract template %s from url %s", fileName, tmpl.getUrl())); } + } else { + throw new CloudRuntimeException(String.format("Failed to download template %s from url %s", fileName, tmpl.getUrl())); } - s_logger.error("Unable to create mount: " + mntPt); - } - - return mountPoint; + }; + THREAD_SERVICE.submit(downloadFile); } - protected boolean mountExists(String localRootPath, URI uri) { - Script script = null; - script = new Script(true, "mount", _timeout, s_logger); - List res = new ArrayList(); - ZfsPathParser parser = new ZfsPathParser(localRootPath); - script.execute(parser); - res.addAll(parser.getPaths()); - for (String s : res) { - if (s.contains(localRootPath)) { - s_logger.debug("Some device already mounted at " + localRootPath + ", no need to mount " + uri.toString()); - return true; - } + private void persistTemplateStoreRef(DataObject template, TemplateDownloader.TemplateInformation info) { + DataStore store = template.getDataStore(); + TemplateDataStoreVO vmTemplateStore = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); + if (vmTemplateStore == null) { + vmTemplateStore = + new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 100, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, info.getInstallPath(), null, null, info.getInstallPath(), template.getUri()); + vmTemplateStore.setDataStoreRole(store.getRole()); + _vmTemplateStoreDao.persist(vmTemplateStore); + } else { + vmTemplateStore.setDownloadPercent(100); + vmTemplateStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + vmTemplateStore.setSize(info.getSize()); + vmTemplateStore.setPhysicalSize(info.getSize()); + vmTemplateStore.setLastUpdated(new Date()); + vmTemplateStore.setInstallPath(info.getInstallPath()); + vmTemplateStore.setLocalDownloadPath(info.getLocalPath()); + vmTemplateStore.setDownloadUrl(template.getUri()); + vmTemplateStore.setState(ObjectInDataStoreStateMachine.State.Ready); + _vmTemplateStoreDao.update(vmTemplateStore.getId(), vmTemplateStore); } - return false; } - public static class ZfsPathParser extends OutputInterpreter { - String _parent; - List paths = new ArrayList(); - public ZfsPathParser(String parent) { - _parent = parent; - } - - @Override - public String interpret(BufferedReader reader) throws IOException { - String line = null; - while ((line = reader.readLine()) != null) { - paths.add(line); + private void persistTemplateStorePoolRef(VirtualMachineTemplate tmpl, long poolId, TemplateDownloader.TemplateInformation info) { + VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, tmpl.getId()); + if (sPoolRef == null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Not found (templateId:" + tmpl.getId() + " poolId: " + poolId + ") in template_spool_ref, persisting it"); } - return null; - } - - public List getPaths() { - return paths; - } - - @Override - public boolean drain() { - return true; + sPoolRef = new VMTemplateStoragePoolVO(poolId, tmpl.getId()); + sPoolRef.setDownloadPercent(100); + sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); + sPoolRef.setTemplateSize(info.getSize()); + sPoolRef.setLocalDownloadPath(info.getInstallPath()); + sPoolRef.setInstallPath(info.getInstallPath()); + vmTemplatePoolDao.persist(sPoolRef); + } else { + sPoolRef.setDownloadPercent(100); + sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); + sPoolRef.setTemplateSize(info.getSize()); + sPoolRef.setLocalDownloadPath(info.getInstallPath()); + sPoolRef.setInstallPath(info.getInstallPath()); + vmTemplatePoolDao.update(sPoolRef.getId(), sPoolRef); } } } diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/HttpTemplateDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/HttpTemplateDownloader.java new file mode 100644 index 000000000000..8fe1a4e7939c --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/managementserver/HttpTemplateDownloader.java @@ -0,0 +1,75 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.storage.download.managementserver; + +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +public class HttpTemplateDownloader extends TemplateDownloaderImpl { + + protected HttpClient client; + public static final Logger logger = Logger.getLogger(HttpTemplateDownloader.class.getName()); + + public HttpTemplateDownloader(VirtualMachineTemplate template, String destPoolPath) { + super(template, destPoolPath); + String downloadDir = getDownloadPath(template.getId()); + createDownloadDirectory(downloadDir); + setDownloadedFilePath(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); + } + + protected void createDownloadDirectory(String downloadDir) { + createFolder(getDestPoolPath() + File.separator + downloadDir); + } + + public boolean downloadTemplate() { + HttpClient client = new DefaultHttpClient(); + HttpGet get = new HttpGet(getTemplate().getUrl()); + try { + HttpResponse response = client.execute(get); + org.apache.http.HttpEntity entity = response.getEntity(); + if (entity == null) { + s_logger.debug("Failed to get entity"); + throw new CloudRuntimeException("Failed to get url: " + getTemplate().getUrl()); + } + File destFile = new File(getDownloadedFilePath()); + if (!destFile.createNewFile()) { + s_logger.warn("Reusing existing file " + destFile.getPath()); + } + try (FileOutputStream outputStream = new FileOutputStream(destFile)) { + entity.writeTo(outputStream); + } catch (IOException e) { + s_logger.debug("downloadFromUrlToNfs:Exception:" + e.getMessage(), e); + } + } catch (IOException e) { + s_logger.debug("Failed to get url:" + getTemplate().getUrl() + ", due to " + e.toString()); + throw new CloudRuntimeException(e); + } + return true; + } +} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloader.java new file mode 100644 index 000000000000..11e890b9584d --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloader.java @@ -0,0 +1,77 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.storage.download.managementserver; + +public interface TemplateDownloader { + + class TemplateInformation { + private String installPath; + private String localPath; + private Long size; + private String checksum; + + public TemplateInformation(String installPath, String localPath, Long size, String checksum) { + this.installPath = installPath; + this.localPath = localPath; + this.size = size; + this.checksum = checksum; + } + + public String getInstallPath() { + return installPath; + } + + public String getLocalPath() { + return localPath; + } + + public Long getSize() { + return size; + } + + public String getChecksum() { + return checksum; + } + } + + /** + * Perform template download to pool specified on downloader creation + * @return true if successful, false if not + */ + boolean downloadTemplate(); + + /** + * Perform extraction (if necessary) and installation of previously downloaded template + * @return true if successful, false if not + */ + boolean extractAndInstallDownloadedTemplate(); + + /** + * Get template information after it is properly installed on pool + * @return template information + */ + TemplateInformation getTemplateInformation(); + + /** + * Perform checksum validation of previously downloadeed template + * @return true if successful, false if not + */ + boolean validateChecksum(); +} diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloaderImpl.java b/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloaderImpl.java new file mode 100644 index 000000000000..0746697f7322 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloaderImpl.java @@ -0,0 +1,231 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.storage.download.managementserver; + +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.util.UUID; + +public abstract class TemplateDownloaderImpl implements TemplateDownloader { + + private String destPoolPath; + private String downloadedFilePath; + private String installPath; + private VirtualMachineTemplate template; + private boolean redownload = false; + private String srcHost; + private String srcPath; + + public static final Logger s_logger = Logger.getLogger(TemplateDownloaderImpl.class.getName()); + private static String systemInstallDir = "template" + File.separator + "tmpl" + File.separator + "1"; + + protected TemplateDownloaderImpl(final VirtualMachineTemplate template, final String destPoolPath) { + this.template = template; + this.destPoolPath = destPoolPath; + parseUrl(); + } + + /** + * Return download path to download template + */ + protected static String getDownloadPath(Long templateId) { + return systemInstallDir + File.separator + templateId; + } + + /** + * Create folder on path if it does not exist + */ + protected void createFolder(String path) { + try { + Files.createDirectories(Paths.get(path)); + } catch (IOException e) { + s_logger.error(String.format("Unable to create directory %s for template %s: %s%n", path, getTemplate().getUrl(), e.toString())); + throw new CloudRuntimeException(e); + } + } + + public String getDestPoolPath() { + return destPoolPath; + } + + + public String getDownloadedFilePath() { + return downloadedFilePath; + } + + public void setDownloadedFilePath(String filePath) { + this.downloadedFilePath = filePath; + } + + public VirtualMachineTemplate getTemplate() { + return template; + } + + public boolean isRedownload() { + return redownload; + } + + /** + * Return filename from url + */ + public String getFileNameFromUrl() { + String[] urlParts = template.getUrl().split("/"); + return urlParts[urlParts.length - 1]; + } + + public String getInstallFileName() { + return template.getUuid() + "." + template.getFormat().toString().toLowerCase(); + } + + /** + * Checks if downloaded template is extractable + * @return true if it should be extracted, false if not + */ + private boolean isTemplateExtractable() { + String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'"); + return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); + } + + @Override + public boolean extractAndInstallDownloadedTemplate() { + installPath = UUID.randomUUID().toString(); + createFolder(getInstallFullPath()); + if (isTemplateExtractable()) { + extractDownloadedTemplate(); + } else { + s_logger.error(String.format("The downloaded template from %s is not extractable.%n", template.getUrl())); + return false; + } + return true; + } + + /** + * Return install full path + */ + private String getInstallFullPath() { + return destPoolPath + File.separator + systemInstallDir + File.separator + String.valueOf(template.getId()); + } + + /** + * Return extract command to execute given downloaded file + */ + private String getExtractCommandForDownloadedFile() { + if (downloadedFilePath.endsWith(".zip")) { + return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath() + File.separator + getInstallFileName(); + } else if (downloadedFilePath.endsWith(".bz2")) { + return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath() + File.separator + getInstallFileName(); + } else if (downloadedFilePath.endsWith(".gz")) { + return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath() + File.separator + getInstallFileName(); + } else { + throw new CloudRuntimeException("Unable to extract template " + template.getId() + " on " + downloadedFilePath); + } + } + + /** + * Extract downloaded template into installPath, remove compressed file + */ + private void extractDownloadedTemplate() { + String extractCommand = getExtractCommandForDownloadedFile(); + Script.runSimpleBashScript(extractCommand); + Script.runSimpleBashScript("rm -f " + downloadedFilePath); + } + + @Override + public TemplateInformation getTemplateInformation() { + String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + File.separator + getInstallFileName() + " | awk '{print $1}'"); + long size = Long.parseLong(sizeResult); + return new TemplateInformation(systemInstallDir + File.separator + String.valueOf(template.getId()) + File.separator + getInstallFileName(), + getTemplate().getUuid(), size, template.getChecksum()); + } + + @Override + public boolean validateChecksum() { + if (StringUtils.isNotBlank(template.getChecksum())) { + int retry = 3; + boolean valid = false; + try { + while (!valid && retry > 0) { + retry--; + s_logger.info("Performing checksum validation for downloaded template " + template.getId() + " using " + template.getChecksum() + ", retries left: " + retry); + valid = DigestHelper.check(template.getChecksum(), new FileInputStream(downloadedFilePath)); + if (!valid && retry > 0) { + s_logger.info("Checksum validation failded, re-downloading template"); + redownload = true; + resetDownloadFile(); + downloadTemplate(); + } + } + s_logger.info("Checksum validation for template " + template.getId() + ": " + (valid ? "succeeded" : "failed")); + return valid; + } catch (IOException e) { + throw new CloudRuntimeException("could not check sum for file: " + downloadedFilePath, e); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException("Unknown checksum algorithm: " + template.getChecksum(), e); + } + } + s_logger.info("No checksum provided, skipping checksum validation"); + return true; + } + + /** + * Delete and create download file + */ + private void resetDownloadFile() { + File f = new File(getDownloadedFilePath()); + s_logger.info("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + template.getId() + " on it"); + try { + if (f.exists()) { + f.delete(); + } + f.createNewFile(); + } catch (IOException e) { + s_logger.error("Error creating file to download on: " + getDownloadedFilePath() + " due to: " + e.getMessage()); + throw new CloudRuntimeException("Failed to create download file for direct download"); + } + } + + /** + * Parse url and set srcHost and srcPath + */ + private void parseUrl() { + try { + URI uri = new URI(UriUtils.encodeURIComponent(template.getUrl())); + if (uri.getScheme() != null && uri.getScheme().equalsIgnoreCase("nfs")) { + srcHost = uri.getHost(); + srcPath = uri.getPath(); + } + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Invalid NFS url " + template.getUrl() + " caused error: " + e.getMessage()); + } + } +} diff --git a/server/src/main/java/com/cloud/storage/mount/MountManager.java b/server/src/main/java/com/cloud/storage/mount/MountManager.java new file mode 100644 index 000000000000..e306cbaf7cd8 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/mount/MountManager.java @@ -0,0 +1,5 @@ +package com.cloud.storage.mount; + +public interface MountManager { + String getMountPoint(String storageUrl, Integer nfsVersion); +} diff --git a/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java b/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java new file mode 100644 index 000000000000..44172f038bdd --- /dev/null +++ b/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java @@ -0,0 +1,194 @@ +package com.cloud.storage.mount; + +import com.cloud.storage.StorageLayer; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.annotation.PreDestroy; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Component +public class NfsMountManager implements MountManager { + private static final Logger s_logger = Logger.getLogger(NfsMountManager.class); + + private StorageLayer storage; + private int timeout; + private final Random rand = new Random(System.currentTimeMillis()); + private final ConcurrentMap storageMounts = new ConcurrentHashMap<>(); + + public static final ConfigKey MOUNT_PARENT = new ConfigKey<>("Advanced", String.class, + "mount.parent", "/var/cloudstack/mnt", + "The mount point on the Management Server for Secondary Storage.", + true, ConfigKey.Scope.Global); + + public NfsMountManager(StorageLayer storage, int timeout) { + this.storage = storage; + this.timeout = timeout; + } + + public String getMountPoint(String storageUrl, Integer nfsVersion) { + String mountPoint = storageMounts.get(storageUrl); + if (mountPoint != null) { + return mountPoint; + } + + URI uri; + try { + uri = new URI(storageUrl); + } catch (URISyntaxException e) { + s_logger.error("Invalid storage URL format ", e); + throw new CloudRuntimeException("Unable to create mount point due to invalid storage URL format " + storageUrl); + } + + mountPoint = mount(uri.getHost() + ":" + uri.getPath(), MOUNT_PARENT.value(), nfsVersion); + if (mountPoint == null) { + s_logger.error("Unable to create mount point for " + storageUrl); + throw new CloudRuntimeException("Unable to create mount point for " + storageUrl); + } + + storageMounts.putIfAbsent(storageUrl, mountPoint); + return mountPoint; + } + + private String mount(String path, String parent, Integer nfsVersion) { + String mountPoint = setupMountPoint(parent); + if (mountPoint == null) { + s_logger.warn("Unable to create a mount point"); + return null; + } + + Script command = new Script(true, "mount", timeout, s_logger); + command.add("-t", "nfs"); + if (nfsVersion != null){ + command.add("-o", "vers=" + nfsVersion); + } + // command.add("-o", "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0"); + if ("Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) { + command.add("-o", "resvport"); + } + command.add(path); + command.add(mountPoint); + String result = command.execute(); + if (result != null) { + s_logger.warn("Unable to mount " + path + " due to " + result); + deleteMountPath(mountPoint); + return null; + } + + // Change permissions for the mountpoint + Script script = new Script(true, "chmod", timeout, s_logger); + script.add("1777", mountPoint); + result = script.execute(); + if (result != null) { + s_logger.warn("Unable to set permissions for " + mountPoint + " due to " + result); + } + return mountPoint; + } + + private String setupMountPoint(String parent) { + String mountPoint = null; + long mshostId = ManagementServerNode.getManagementServerId(); + for (int i = 0; i < 10; i++) { + String mntPt = parent + File.separator + String.valueOf(mshostId) + "." + Integer.toHexString(rand.nextInt(Integer.MAX_VALUE)); + File file = new File(mntPt); + if (!file.exists()) { + if (storage.mkdir(mntPt)) { + mountPoint = mntPt; + break; + } + } + s_logger.error("Unable to create mount: " + mntPt); + } + + return mountPoint; + } + + private void umount(String localRootPath) { + if (!mountExists(localRootPath)) { + return; + } + Script command = new Script(true, "umount", timeout, s_logger); + command.add(localRootPath); + String result = command.execute(); + if (result != null) { + // Fedora Core 12 errors out with any -o option executed from java + String errMsg = "Unable to umount " + localRootPath + " due to " + result; + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + deleteMountPath(localRootPath); + s_logger.debug("Successfully umounted " + localRootPath); + } + + private void deleteMountPath(String localRootPath) { + try { + Files.deleteIfExists(Paths.get(localRootPath)); + } catch (IOException e) { + s_logger.warn(String.format("unable to delete mount directory %s:%s.%n", localRootPath, e.getMessage())); + } + } + + private boolean mountExists(String localRootPath) { + Script script = new Script(true, "mount", timeout, s_logger); +// List res = new ArrayList<>(); + ZfsPathParser parser = new ZfsPathParser(localRootPath); + script.execute(parser); +// res.addAll(parser.getPaths()); + return parser.getPaths().stream().filter(s -> s.contains(localRootPath)).findAny().map(s -> true).orElse(false); +// for (String s : res) { +// if (s.contains(localRootPath)) { +// s_logger.debug("Some device already mounted at " + localRootPath); +// return true; +// } +// } +// return false; + } + + public static class ZfsPathParser extends OutputInterpreter { + String _parent; + List paths = new ArrayList<>(); + + public ZfsPathParser(String parent) { + _parent = parent; + } + + @Override + public String interpret(BufferedReader reader) throws IOException { + String line; + while ((line = reader.readLine()) != null) { + paths.add(line); + } + return null; + } + + public List getPaths() { + return paths; + } + + @Override + public boolean drain() { + return true; + } + } + + @PreDestroy + public void destroy() { + storageMounts.values().stream().forEach(this::umount); + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 965a9b8ab426..8eb56b3f2d13 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -65,6 +65,12 @@ + + + + + + From 61e34245213d887bcde5bff01a18f0def1a5d98d Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 7 Nov 2018 16:48:25 +0200 Subject: [PATCH 11/13] add ssvm-agnostic capacity checking and properties file creation --- .../storage/download/DownloadMonitorImpl.java | 2 +- .../ManagementServerDownloader.java | 27 ++++-- .../HttpSystemTemplateDownloader.java} | 16 ++- .../SystemTemplateDownloader.java} | 50 ++++++++-- .../cloud/storage/mount/NfsMountManager.java | 13 +-- .../storage/secondary/CapacityChecker.java | 18 ++++ .../secondary/NfsMountCapacityChecker.java | 51 ++++++++++ .../template/HypervisorTemplateAdapter.java | 97 ++++++++++--------- .../spring-server-core-managers-context.xml | 3 +- 9 files changed, 195 insertions(+), 82 deletions(-) rename server/src/main/java/com/cloud/storage/download/{ => managementserver}/ManagementServerDownloader.java (83%) rename server/src/main/java/com/cloud/storage/download/managementserver/{HttpTemplateDownloader.java => system/HttpSystemTemplateDownloader.java} (83%) rename server/src/main/java/com/cloud/storage/download/managementserver/{TemplateDownloaderImpl.java => system/SystemTemplateDownloader.java} (75%) create mode 100644 server/src/main/java/com/cloud/storage/secondary/CapacityChecker.java create mode 100644 server/src/main/java/com/cloud/storage/secondary/NfsMountCapacityChecker.java diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index d26c68b486c3..7f0780d9a54f 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -27,6 +27,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.download.managementserver.ManagementServerDownloader; import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.upload.UploadListener; import com.cloud.template.VirtualMachineTemplate; @@ -156,7 +157,6 @@ private void initiateTemplateDownload(DataObject template, AsyncCompletionCallba dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd); } EndPoint ep = _epSelector.select(template); - ep = null;//just for test -> remove if (ep == null) { if (tmpl.getTemplateType() == Storage.TemplateType.SYSTEM) { managementServerDownloader.download(tmpl, template); diff --git a/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java similarity index 83% rename from server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java rename to server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java index b4576d02aeb4..997ae0ea9a18 100644 --- a/server/src/main/java/com/cloud/storage/download/ManagementServerDownloader.java +++ b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java @@ -1,11 +1,12 @@ -package com.cloud.storage.download; +package com.cloud.storage.download.managementserver; import com.cloud.storage.ImageStoreDetailsUtil; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.storage.download.managementserver.HttpTemplateDownloader; -import com.cloud.storage.download.managementserver.TemplateDownloader; +import com.cloud.storage.download.managementserver.system.HttpSystemTemplateDownloader; import com.cloud.storage.mount.MountManager; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -20,6 +21,7 @@ import javax.inject.Inject; import java.util.Date; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -33,7 +35,10 @@ public class ManagementServerDownloader { VMTemplatePoolDao vmTemplatePoolDao; @Inject - private TemplateDataStoreDao _vmTemplateStoreDao; + private VMTemplateDao templateDao; + + @Inject + private TemplateDataStoreDao vmTemplateStoreDao; @Inject private MountManager mountManager; @@ -49,7 +54,9 @@ public void downloadFromUrlToNfs(VirtualMachineTemplate tmpl, DataObject templat Runnable downloadFile = () -> { Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(template.getDataStore().getId()); String mountPoint = mountManager.getMountPoint(template.getDataStore().getUri(), nfsVersion); - HttpTemplateDownloader downloader = new HttpTemplateDownloader(tmpl, mountPoint); + VMTemplateVO templateVO = Optional.ofNullable(templateDao.findById(template.getId())) + .orElseThrow(() -> new CloudRuntimeException(String.format("Unable to find template by id %d.%n", template.getId()))); + HttpSystemTemplateDownloader downloader = new HttpSystemTemplateDownloader(tmpl, templateVO, mountPoint); if (downloader.downloadTemplate()) { if (downloader.extractAndInstallDownloadedTemplate()) { TemplateDownloader.TemplateInformation info = downloader.getTemplateInformation(); @@ -69,12 +76,12 @@ public void downloadFromUrlToNfs(VirtualMachineTemplate tmpl, DataObject templat private void persistTemplateStoreRef(DataObject template, TemplateDownloader.TemplateInformation info) { DataStore store = template.getDataStore(); - TemplateDataStoreVO vmTemplateStore = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); + TemplateDataStoreVO vmTemplateStore = vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); if (vmTemplateStore == null) { vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 100, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, info.getInstallPath(), null, null, info.getInstallPath(), template.getUri()); vmTemplateStore.setDataStoreRole(store.getRole()); - _vmTemplateStoreDao.persist(vmTemplateStore); + vmTemplateStoreDao.persist(vmTemplateStore); } else { vmTemplateStore.setDownloadPercent(100); vmTemplateStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); @@ -85,7 +92,7 @@ private void persistTemplateStoreRef(DataObject template, TemplateDownloader.Tem vmTemplateStore.setLocalDownloadPath(info.getLocalPath()); vmTemplateStore.setDownloadUrl(template.getUri()); vmTemplateStore.setState(ObjectInDataStoreStateMachine.State.Ready); - _vmTemplateStoreDao.update(vmTemplateStore.getId(), vmTemplateStore); + vmTemplateStoreDao.update(vmTemplateStore.getId(), vmTemplateStore); } } @@ -100,7 +107,7 @@ private void persistTemplateStorePoolRef(VirtualMachineTemplate tmpl, long poolI sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); sPoolRef.setTemplateSize(info.getSize()); - sPoolRef.setLocalDownloadPath(info.getInstallPath()); + sPoolRef.setLocalDownloadPath(info.getLocalPath()); sPoolRef.setInstallPath(info.getInstallPath()); vmTemplatePoolDao.persist(sPoolRef); } else { @@ -108,7 +115,7 @@ private void persistTemplateStorePoolRef(VirtualMachineTemplate tmpl, long poolI sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); sPoolRef.setTemplateSize(info.getSize()); - sPoolRef.setLocalDownloadPath(info.getInstallPath()); + sPoolRef.setLocalDownloadPath(info.getLocalPath()); sPoolRef.setInstallPath(info.getInstallPath()); vmTemplatePoolDao.update(sPoolRef.getId(), sPoolRef); } diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/HttpTemplateDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/system/HttpSystemTemplateDownloader.java similarity index 83% rename from server/src/main/java/com/cloud/storage/download/managementserver/HttpTemplateDownloader.java rename to server/src/main/java/com/cloud/storage/download/managementserver/system/HttpSystemTemplateDownloader.java index 8fe1a4e7939c..3a10c9af2e86 100644 --- a/server/src/main/java/com/cloud/storage/download/managementserver/HttpTemplateDownloader.java +++ b/server/src/main/java/com/cloud/storage/download/managementserver/system/HttpSystemTemplateDownloader.java @@ -17,8 +17,10 @@ // under the License. // -package com.cloud.storage.download.managementserver; +package com.cloud.storage.download.managementserver.system; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.http.HttpResponse; @@ -27,17 +29,21 @@ import org.apache.http.impl.client.DefaultHttpClient; import org.apache.log4j.Logger; +import javax.inject.Inject; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -public class HttpTemplateDownloader extends TemplateDownloaderImpl { +public class HttpSystemTemplateDownloader extends SystemTemplateDownloader { + + @Inject + private VMTemplateDao _templateDao; protected HttpClient client; - public static final Logger logger = Logger.getLogger(HttpTemplateDownloader.class.getName()); + public static final Logger logger = Logger.getLogger(HttpSystemTemplateDownloader.class.getName()); - public HttpTemplateDownloader(VirtualMachineTemplate template, String destPoolPath) { - super(template, destPoolPath); + public HttpSystemTemplateDownloader(VirtualMachineTemplate template,VMTemplateVO templateVO, String destPoolPath) { + super(template, templateVO, destPoolPath); String downloadDir = getDownloadPath(template.getId()); createDownloadDirectory(downloadDir); setDownloadedFilePath(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloaderImpl.java b/server/src/main/java/com/cloud/storage/download/managementserver/system/SystemTemplateDownloader.java similarity index 75% rename from server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloaderImpl.java rename to server/src/main/java/com/cloud/storage/download/managementserver/system/SystemTemplateDownloader.java index 0746697f7322..b68237924ec3 100644 --- a/server/src/main/java/com/cloud/storage/download/managementserver/TemplateDownloaderImpl.java +++ b/server/src/main/java/com/cloud/storage/download/managementserver/system/SystemTemplateDownloader.java @@ -16,8 +16,10 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.storage.download.managementserver; +package com.cloud.storage.download.managementserver.system; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.download.managementserver.TemplateDownloader; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; @@ -32,25 +34,30 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; +import java.util.Optional; import java.util.UUID; -public abstract class TemplateDownloaderImpl implements TemplateDownloader { +public abstract class SystemTemplateDownloader implements TemplateDownloader { private String destPoolPath; private String downloadedFilePath; private String installPath; private VirtualMachineTemplate template; + private VMTemplateVO templateVO; private boolean redownload = false; private String srcHost; private String srcPath; - public static final Logger s_logger = Logger.getLogger(TemplateDownloaderImpl.class.getName()); + public static final Logger s_logger = Logger.getLogger(SystemTemplateDownloader.class.getName()); private static String systemInstallDir = "template" + File.separator + "tmpl" + File.separator + "1"; + private static final String systemPropertiesFileName = "template.properties"; - protected TemplateDownloaderImpl(final VirtualMachineTemplate template, final String destPoolPath) { + protected SystemTemplateDownloader(VirtualMachineTemplate template, VMTemplateVO templateVO, String destPoolPath) { this.template = template; + this.templateVO = templateVO; this.destPoolPath = destPoolPath; parseUrl(); } @@ -123,12 +130,41 @@ public boolean extractAndInstallDownloadedTemplate() { if (isTemplateExtractable()) { extractDownloadedTemplate(); } else { - s_logger.error(String.format("The downloaded template from %s is not extractable.%n", template.getUrl())); - return false; + s_logger.info(String.format("The downloaded template from %s is not extractable.%n", template.getUrl())); } + createPropertiesFile(); return true; } + private void createPropertiesFile() { + try { + Path propertiesFile = Files.createFile(Paths.get(getInstallFullPath() + File.separator + systemPropertiesFileName)); + Files.write(propertiesFile, getTemplatePropertiesFileContent().getBytes()); + } catch (IOException e) { + String errorMessage = String.format("Unable to create %s file for downloaded template %s: %s.%n", systemPropertiesFileName, template.getUrl(), e.getMessage()); + s_logger.error(errorMessage); + throw new CloudRuntimeException(errorMessage); + } + } + + private String getTemplatePropertiesFileContent() { + TemplateInformation info = getTemplateInformation(); + StringBuffer content = new StringBuffer(500); + content.append(String.format("filename=%s%n", getInstallFileName())); + content.append(String.format("description=%s%n", template.getDisplayText())); + content.append(String.format("checksum=%s%n", Optional.ofNullable(template.getChecksum()).orElse(""))); + content.append(String.format("hvm=%s%n", template.isRequiresHvm())); + content.append(String.format("size=%s%n", info.getSize())); + content.append(String.format("%s=true%n", template.getFormat().getFileExtension())); + content.append(String.format("id=%s%n", template.getId())); + content.append(String.format("public=%s%n", template.isPublicTemplate())); + content.append(String.format("%s.filename=%s%n", template.getFormat().getFileExtension(), getInstallFileName())); + content.append(String.format("uniquename=%s%n", templateVO.getUniqueName())); + content.append(String.format("%s.virtualsize=%s%n", template.getFormat().getFileExtension(), info.getSize())); + content.append(String.format("%s.size=%s%n", template.getFormat().getFileExtension(), info.getSize())); + return content.toString(); + } + /** * Return install full path */ @@ -164,7 +200,7 @@ private void extractDownloadedTemplate() { public TemplateInformation getTemplateInformation() { String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + File.separator + getInstallFileName() + " | awk '{print $1}'"); long size = Long.parseLong(sizeResult); - return new TemplateInformation(systemInstallDir + File.separator + String.valueOf(template.getId()) + File.separator + getInstallFileName(), + return new TemplateInformation(systemInstallDir + File.separator + String.valueOf(template.getId()), getTemplate().getUuid(), size, template.getChecksum()); } diff --git a/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java b/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java index 44172f038bdd..62dd7f545623 100644 --- a/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java +++ b/server/src/main/java/com/cloud/storage/mount/NfsMountManager.java @@ -103,9 +103,8 @@ private String mount(String path, String parent, Integer nfsVersion) { private String setupMountPoint(String parent) { String mountPoint = null; - long mshostId = ManagementServerNode.getManagementServerId(); for (int i = 0; i < 10; i++) { - String mntPt = parent + File.separator + String.valueOf(mshostId) + "." + Integer.toHexString(rand.nextInt(Integer.MAX_VALUE)); + String mntPt = parent + File.separator + String.valueOf(ManagementServerNode.getManagementServerId()) + "." + Integer.toHexString(rand.nextInt(Integer.MAX_VALUE)); File file = new File(mntPt); if (!file.exists()) { if (storage.mkdir(mntPt)) { @@ -146,18 +145,9 @@ private void deleteMountPath(String localRootPath) { private boolean mountExists(String localRootPath) { Script script = new Script(true, "mount", timeout, s_logger); -// List res = new ArrayList<>(); ZfsPathParser parser = new ZfsPathParser(localRootPath); script.execute(parser); -// res.addAll(parser.getPaths()); return parser.getPaths().stream().filter(s -> s.contains(localRootPath)).findAny().map(s -> true).orElse(false); -// for (String s : res) { -// if (s.contains(localRootPath)) { -// s_logger.debug("Some device already mounted at " + localRootPath); -// return true; -// } -// } -// return false; } public static class ZfsPathParser extends OutputInterpreter { @@ -189,6 +179,7 @@ public boolean drain() { @PreDestroy public void destroy() { + s_logger.info("Clean up mounted NFS mount points used in current session."); storageMounts.values().stream().forEach(this::umount); } } diff --git a/server/src/main/java/com/cloud/storage/secondary/CapacityChecker.java b/server/src/main/java/com/cloud/storage/secondary/CapacityChecker.java new file mode 100644 index 000000000000..2003fcbfecf7 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/secondary/CapacityChecker.java @@ -0,0 +1,18 @@ +package com.cloud.storage.secondary; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; + +public interface CapacityChecker { + double CAPACITY_THRESHOLD = 0.90; + + boolean hasEnoughCapacity(DataStore imageStore); + + default Long parse(String value) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + throw new CloudRuntimeException(String.format("unable to parse %s: %s.%n", value, e.getMessage())); + } + } +} diff --git a/server/src/main/java/com/cloud/storage/secondary/NfsMountCapacityChecker.java b/server/src/main/java/com/cloud/storage/secondary/NfsMountCapacityChecker.java new file mode 100644 index 000000000000..1bc731dcd3a5 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/secondary/NfsMountCapacityChecker.java @@ -0,0 +1,51 @@ +package com.cloud.storage.secondary; + +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.mount.MountManager; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Optional; + +@Component +public class NfsMountCapacityChecker implements CapacityChecker { + @Inject + private MountManager mountManager; + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; + + @Override + public boolean hasEnoughCapacity(DataStore imageStore) { + String[] capacityInfo = getCapacityInfo(imageStore); + Long capacity = parse(capacityInfo[0]); + Long used = parse(capacityInfo[1]); + if (used/(capacity * 1.0) <= CAPACITY_THRESHOLD) { + return true; + } + return false; + } + + private String[] getCapacityInfo(DataStore imageStore) { + String mountPoint = mountManager.getMountPoint(imageStore.getUri(), imageStoreDetailsUtil.getNfsVersion(imageStore.getId())); + String command = String.format("df %s | grep %s | awk -F' ' '{print $2, $3}'", mountPoint, hostFromUrl(imageStore.getUri())); + return Optional.ofNullable(Script.runSimpleBashScript(command)).orElse("0 0").split(" "); + } + + private String hostFromUrl(String url) { + try { + URI uri = new URI(UriUtils.encodeURIComponent(url)); + if (Optional.ofNullable(uri.getScheme()).orElse("").equalsIgnoreCase("nfs")) { + return uri.getHost(); + } + return url; + } catch (URISyntaxException e) { + throw new CloudRuntimeException(String.format("Invalid NFS url %s caused error: %s.%n", url, e.getMessage())); + } + } +} diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index ac78e84116c5..814c7d174b51 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -16,38 +16,55 @@ // under the License. package com.cloud.template; +import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Config; +import com.cloud.configuration.Resource.ResourceType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.org.Grouping; import com.cloud.resource.ResourceManager; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ExecutionException; - -import javax.inject.Inject; - -import com.cloud.configuration.Config; +import com.cloud.server.StatsCollector; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.TemplateProfile; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.storage.download.DownloadMonitor; +import com.cloud.storage.secondary.CapacityChecker; +import com.cloud.template.VirtualMachineTemplate.State; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; -import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.log4j.Logger; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -61,37 +78,20 @@ import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.log4j.Logger; -import com.cloud.agent.AgentManager; -import com.cloud.alert.AlertManager; -import com.cloud.configuration.Resource.ResourceType; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.dao.DataCenterDao; -import com.cloud.event.EventTypes; -import com.cloud.event.UsageEventUtils; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.org.Grouping; -import com.cloud.server.StatsCollector; -import com.cloud.template.VirtualMachineTemplate.State; -import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.TemplateType; -import com.cloud.storage.TemplateProfile; -import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.VMTemplateZoneVO; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplateZoneDao; -import com.cloud.storage.download.DownloadMonitor; -import com.cloud.utils.UriUtils; -import com.cloud.utils.db.DB; -import com.cloud.utils.db.EntityManager; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; public class HypervisorTemplateAdapter extends TemplateAdapterBase { private final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); @@ -125,6 +125,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { ResourceManager resourceManager; @Inject VMTemplateDao templateDao; + @Inject + CapacityChecker capacityChecker; @Override public String getName() { @@ -251,13 +253,14 @@ private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTempl } // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + if (Grouping.AllocationState.Disabled == zone.getAllocationState() && !template.getTemplateType().equals(Storage.TemplateType.SYSTEM)) { s_logger.info("Zone " + zoneId + " is disabled. Skip downloading template to its image store " + imageStore.getId()); continue; } // Check if image store has enough capacity for template - if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore) && + (template.getTemplateType().equals(Storage.TemplateType.SYSTEM) && !capacityChecker.hasEnoughCapacity(imageStore))) { s_logger.info("Image store doesn't have enough capacity. Skip downloading template to this image store " + imageStore.getId()); continue; } diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 8eb56b3f2d13..b1c809d2e3b2 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -70,6 +70,7 @@ + - + From 506715211551835768c9b90f93cb19a78ec15bd2 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 14 Nov 2018 11:26:09 +0200 Subject: [PATCH 12/13] add system vm template copy logic --- .../user/template/RegisterTemplateCmd.java | 2 +- .../api/storage/TemplateService.java | 2 + .../storage/image/TemplateServiceImpl.java | 38 +++- .../spring-engine-storage-core-context.xml | 3 + .../storage/copy/ManagementServerCopier.java | 10 + .../copy/NfsManagementServerCopier.java | 71 +++++++ .../ManagementServerDownloader.java | 4 +- .../cloud/template/TemplateManagerImpl.java | 200 +++++++++--------- 8 files changed, 220 insertions(+), 110 deletions(-) create mode 100644 server/src/main/java/com/cloud/storage/copy/ManagementServerCopier.java create mode 100644 server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 81a731e2ad89..e7b51d590713 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -166,7 +166,7 @@ public class RegisterTemplateCmd extends BaseCmd { private Boolean system; @Parameter(name=ApiConstants.ACTIVATE, type = CommandType.BOOLEAN, - description = "true if this template should be used by CloudStack to create System VMs. Must be used with templatetype of 'system'.") + description = "true if this template should be used by CloudStack to create System VMs. Must be used with template type of 'system'.") private Boolean activate; ///////////////////////////////////////////////////// diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index fc8a769e81f9..abad08aeb795 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -51,6 +51,8 @@ public TemplateInfo getTemplate() { AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, DataStore destStore); + AsyncCallFuture copySystemVMTemplate(TemplateInfo srcTemplate, DataStore destStore); + AsyncCallFuture prepareTemplateOnPrimary(TemplateInfo srcTemplate, StoragePool pool); AsyncCallFuture deleteTemplateOnPrimary(TemplateInfo template, StoragePool pool); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 45e3941a5ec3..37ab7accbdd1 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; +import com.cloud.storage.copy.ManagementServerCopier; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; @@ -156,6 +157,8 @@ public class TemplateServiceImpl implements TemplateService { ImageStoreDetailsUtil imageStoreDetailsUtil; @Inject TemplateDataFactory imageFactory; + @Inject + private ManagementServerCopier managementServerCopier; class TemplateOpContext extends AsyncRpcContext { final TemplateObject template; @@ -1017,6 +1020,9 @@ public void syncTemplateToRegionStore(long templateId, DataStore store) { @Override public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, DataStore destStore) { + if (srcTemplate.getTemplateType() == TemplateType.SYSTEM) { + return copySystemVMTemplate(srcTemplate, destStore); + } // for vmware template, we need to check if ova packing is needed, since template created from snapshot does not have .ova file // we invoke createEntityExtractURL to trigger ova packing. Ideally, we can directly use extractURL to pass to following createTemplate. // Need to understand what is the background to use two different urls for copy and extract. @@ -1055,11 +1061,7 @@ public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, destStore.getDriver().createAsync(destStore, templateOnStore, caller); } catch (CloudRuntimeException ex) { // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called - TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), srcTemplate.getId()); - if (templateStoreVO != null) { - TemplateInfo tmplObj = _templateFactory.getTemplate(srcTemplate, destStore); - tmplObj.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); - } + cleanupTemplateStoreRefEntry(srcTemplate, destStore); TemplateApiResult res = new TemplateApiResult((TemplateObject)templateOnStore); res.setResult(ex.getMessage()); future.complete(res); @@ -1067,6 +1069,32 @@ public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, return future; } + @Override + public AsyncCallFuture copySystemVMTemplate(TemplateInfo srcTemplate, DataStore destStore) { + AsyncCallFuture future = new AsyncCallFuture<>(); + TemplateObject tmplForCopy = (TemplateObject)_templateFactory.getTemplate(srcTemplate, destStore); + DataObject templateOnStore = destStore.create(tmplForCopy); + templateOnStore.processEvent(Event.CreateOnlyRequested); + TemplateApiResult res = new TemplateApiResult(new TemplateObject()); + try { + res.setSuccess(managementServerCopier.copy(srcTemplate, destStore)); + } catch (CloudRuntimeException e) { + cleanupTemplateStoreRefEntry(srcTemplate, destStore); + res.setSuccess(false); + res.setResult(e.getMessage()); + } + future.complete(res); + return future; + } + + private void cleanupTemplateStoreRefEntry(TemplateInfo srcTemplate, DataStore destStore) { + TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), srcTemplate.getId()); + if (templateStoreVO != null) { + TemplateInfo info = _templateFactory.getTemplate(srcTemplate, destStore); + info.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + } + } + private String generateCopyUrl(String ipAddress, String dir, String path) { String hostname = ipAddress; String scheme = "http"; diff --git a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml index 33385b5ae001..f512272f862d 100644 --- a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml +++ b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml @@ -50,6 +50,9 @@ + + diff --git a/server/src/main/java/com/cloud/storage/copy/ManagementServerCopier.java b/server/src/main/java/com/cloud/storage/copy/ManagementServerCopier.java new file mode 100644 index 000000000000..bc9b75662a53 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/copy/ManagementServerCopier.java @@ -0,0 +1,10 @@ +package com.cloud.storage.copy; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; + + +public interface ManagementServerCopier { + + boolean copy(TemplateInfo srcTemplate, DataStore destStore); +} diff --git a/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java b/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java new file mode 100644 index 000000000000..5b80d337fa28 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java @@ -0,0 +1,71 @@ +package com.cloud.storage.copy; + +import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.mount.MountManager; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; + +@Component +public class NfsManagementServerCopier implements ManagementServerCopier { + private static final Logger s_logger = Logger.getLogger(NfsManagementServerCopier.class); + + @Inject + private MountManager mountManager; + @Inject + private ImageStoreDetailsUtil imageStoreDetailsUtil; + @Inject + private TemplateDataStoreDao vmTemplateStoreDao; + + + public boolean copy(TemplateInfo srcTemplate, DataStore destStore) { + Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(srcTemplate.getDataStore().getId()); + String sourceMountPoint = mountManager.getMountPoint(srcTemplate.getDataStore().getUri(), nfsVersion); + String destMountPoint = mountManager.getMountPoint(destStore.getUri(), nfsVersion); + String installPath = getTemplateDataStore(srcTemplate).getInstallPath(); + createFolder(Paths.get(destMountPoint, installPath).getParent()); + String sourcePath = Paths.get(sourceMountPoint, installPath).getParent().toString(); + String destinationPath = Paths.get(destMountPoint, installPath).getParent().toString(); + return copyFolderContent(sourcePath, destinationPath); + } + + private boolean copyFolderContent(String sourcePath, String destinationPath) { + String command = String.format("cp %s%s* %s", sourcePath, File.separator, destinationPath); + String result = Script.runSimpleBashScript(command); + if (result != null) { + s_logger.warn(String.format("Unable to copy from %s to %s due to %s.%n", sourcePath, destinationPath, result)); + return false; + } + return true; + } + + /** + * Create folder on path if it does not exist + */ + public static void createFolder(Path path) { + try { + Files.createDirectories(path); + } catch (IOException e) { + s_logger.error(String.format("Unable to create directory from path %s: %s.%n", path, e.toString())); + throw new CloudRuntimeException(e); + } + } + + private TemplateDataStoreVO getTemplateDataStore(TemplateInfo template) { + return Optional.ofNullable(vmTemplateStoreDao.findByStoreTemplate(template.getDataStore().getId(), template.getId())) + .orElseThrow(() -> new CloudRuntimeException(String.format("Unable to find template store ref by template id %d.%n", template.getId()))); + } +} diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java index 997ae0ea9a18..e86ae42513ae 100644 --- a/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java +++ b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java @@ -78,8 +78,8 @@ private void persistTemplateStoreRef(DataObject template, TemplateDownloader.Tem DataStore store = template.getDataStore(); TemplateDataStoreVO vmTemplateStore = vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); if (vmTemplateStore == null) { - vmTemplateStore = - new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 100, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, info.getInstallPath(), null, null, info.getInstallPath(), template.getUri()); + vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 100, + VMTemplateStorageResourceAssoc.Status.DOWNLOADED, info.getInstallPath(), null, null, info.getInstallPath(), template.getUri()); vmTemplateStore.setDataStoreRole(store.getRole()); vmTemplateStoreDao.persist(vmTemplateStore); } else { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 8858367b6232..d34a8a3d1974 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -16,107 +16,6 @@ // under the License. package com.cloud.template; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import com.cloud.deploy.DeployDestination; -import com.cloud.storage.ImageStoreUploadMonitorImpl; -import com.cloud.upgrade.dao.VersionDao; -import com.cloud.utils.StringUtils; -import com.cloud.utils.EncryptionUtil; -import com.cloud.utils.DateUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.EnumUtils; -import com.google.common.base.Joiner; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; -import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; -import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.async.AsyncCallFuture; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.apache.log4j.Logger; -import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; -import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd; -import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; -import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; -import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; -import org.apache.cloudstack.api.command.user.iso.ListIsoPermissionsCmd; -import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; -import org.apache.cloudstack.api.command.user.iso.UpdateIsoCmd; -import org.apache.cloudstack.api.command.user.iso.UpdateIsoPermissionsCmd; -import org.apache.cloudstack.api.command.user.template.CopyTemplateCmd; -import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; -import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; -import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; -import org.apache.cloudstack.api.response.GetUploadParamsResponse; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.Scope; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.framework.messagebus.PublishScope; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CommandResult; -import org.apache.cloudstack.storage.command.DettachCommand; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; -import org.apache.cloudstack.storage.to.TemplateObjectTO; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -134,6 +33,7 @@ import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeployDestination; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; @@ -153,6 +53,7 @@ import com.cloud.projects.ProjectManager; import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; +import com.cloud.storage.ImageStoreUploadMonitorImpl; import com.cloud.storage.LaunchPermissionVO; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -184,6 +85,7 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.template.TemplateAdapter.TemplateAdapterType; import com.cloud.template.VirtualMachineTemplate.BootloaderType; +import com.cloud.upgrade.dao.VersionDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -191,6 +93,11 @@ import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.uservm.UserVm; +import com.cloud.utils.DateUtil; +import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.EnumUtils; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -207,10 +114,99 @@ import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; - +import com.google.common.base.Joiner; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; +import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd; +import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; +import org.apache.cloudstack.api.command.admin.template.GetSystemVMTemplateDefaultURLCmd; +import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; +import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; +import org.apache.cloudstack.api.command.user.iso.ListIsoPermissionsCmd; +import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; +import org.apache.cloudstack.api.command.user.iso.UpdateIsoCmd; +import org.apache.cloudstack.api.command.user.iso.UpdateIsoPermissionsCmd; +import org.apache.cloudstack.api.command.user.template.CopyTemplateCmd; +import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; +import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; +import org.apache.cloudstack.api.response.GetSystemVMTemplateDefaultURLResponse; +import org.apache.cloudstack.api.response.GetUploadParamsResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.DettachCommand; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateHyperV; import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateKvm; import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterTemplateLxc; From eb48788cbd4afaeb54cd5148efb3409a32ee0287 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 14 Nov 2018 16:32:59 +0200 Subject: [PATCH 13/13] add persistence to system vm template copy logic --- .../copy/NfsManagementServerCopier.java | 36 ++++++++++++++++--- .../ManagementServerDownloader.java | 6 ++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java b/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java index 5b80d337fa28..5f26b3ce5fb8 100644 --- a/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java +++ b/server/src/main/java/com/cloud/storage/copy/NfsManagementServerCopier.java @@ -1,10 +1,12 @@ package com.cloud.storage.copy; import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.mount.MountManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; @@ -17,6 +19,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Date; import java.util.Optional; @Component @@ -36,10 +39,14 @@ public boolean copy(TemplateInfo srcTemplate, DataStore destStore) { String sourceMountPoint = mountManager.getMountPoint(srcTemplate.getDataStore().getUri(), nfsVersion); String destMountPoint = mountManager.getMountPoint(destStore.getUri(), nfsVersion); String installPath = getTemplateDataStore(srcTemplate).getInstallPath(); - createFolder(Paths.get(destMountPoint, installPath).getParent()); - String sourcePath = Paths.get(sourceMountPoint, installPath).getParent().toString(); - String destinationPath = Paths.get(destMountPoint, installPath).getParent().toString(); - return copyFolderContent(sourcePath, destinationPath); + String sourcePath = Paths.get(sourceMountPoint, installPath).toString(); + String destinationPath = Paths.get(destMountPoint, installPath).toString(); + createFolder(Paths.get(destMountPoint, installPath)); + if (copyFolderContent(sourcePath, destinationPath)) { + persistTemplateStoreRef(srcTemplate, destStore, destinationPath); + return true; + } + return false; } private boolean copyFolderContent(String sourcePath, String destinationPath) { @@ -68,4 +75,25 @@ private TemplateDataStoreVO getTemplateDataStore(TemplateInfo template) { return Optional.ofNullable(vmTemplateStoreDao.findByStoreTemplate(template.getDataStore().getId(), template.getId())) .orElseThrow(() -> new CloudRuntimeException(String.format("Unable to find template store ref by template id %d.%n", template.getId()))); } + + private void persistTemplateStoreRef(TemplateInfo template, DataStore destStore, String destinationPath) { + TemplateDataStoreVO vmTemplateStore = vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), template.getId()); + if (vmTemplateStore == null) { + vmTemplateStore = new TemplateDataStoreVO(destStore.getId(), template.getId(), new Date(), 100, + VMTemplateStorageResourceAssoc.Status.DOWNLOADED, template.getInstallPath(), null, null, template.getInstallPath(), template.getUri()); + vmTemplateStore.setDataStoreRole(destStore.getRole()); + vmTemplateStoreDao.persist(vmTemplateStore); + } else { + vmTemplateStore.setDownloadPercent(100); + vmTemplateStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + vmTemplateStore.setSize(Optional.ofNullable(template.getSize()).orElse(0L)); + vmTemplateStore.setPhysicalSize(Optional.ofNullable(template.getSize()).orElse(0L)); + vmTemplateStore.setLastUpdated(new Date()); + vmTemplateStore.setInstallPath(template.getInstallPath()); + vmTemplateStore.setLocalDownloadPath(destinationPath); + vmTemplateStore.setDownloadUrl(template.getUri()); + vmTemplateStore.setState(ObjectInDataStoreStateMachine.State.Ready); + vmTemplateStoreDao.update(vmTemplateStore.getId(), vmTemplateStore); + } + } } diff --git a/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java index e86ae42513ae..0bbd8a09a947 100644 --- a/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java +++ b/server/src/main/java/com/cloud/storage/download/managementserver/ManagementServerDownloader.java @@ -61,6 +61,7 @@ public void downloadFromUrlToNfs(VirtualMachineTemplate tmpl, DataObject templat if (downloader.extractAndInstallDownloadedTemplate()) { TemplateDownloader.TemplateInformation info = downloader.getTemplateInformation(); try(TransactionLegacy txn = TransactionLegacy.open(ManagementServerDownloader.class.getName())) { + persistTemplate(template, templateVO, info); persistTemplateStorePoolRef(tmpl, template.getDataStore().getId(), info); persistTemplateStoreRef(template, info); } @@ -74,6 +75,11 @@ public void downloadFromUrlToNfs(VirtualMachineTemplate tmpl, DataObject templat THREAD_SERVICE.submit(downloadFile); } + private void persistTemplate(DataObject template, VMTemplateVO templateVO, TemplateDownloader.TemplateInformation info) { + templateVO.setSize(info.getSize()); + templateDao.update(template.getId(), templateVO); + } + private void persistTemplateStoreRef(DataObject template, TemplateDownloader.TemplateInformation info) { DataStore store = template.getDataStore(); TemplateDataStoreVO vmTemplateStore = vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId());